2024-11-18 18:18:29,891 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 2024-11-18 18:18:30,046 main DEBUG Took 0.043612 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-18 18:18:30,046 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-18 18:18:30,047 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-18 18:18:30,048 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-18 18:18:30,050 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:18:30,084 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-18 18:18:30,154 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:18:30,166 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:18:30,168 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:18:30,178 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:18:30,182 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:18:30,183 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:18:30,187 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:18:30,187 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:18:30,189 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:18:30,190 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:18:30,191 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:18:30,200 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:18:30,201 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:18:30,202 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:18:30,215 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:18:30,215 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:18:30,216 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:18:30,216 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:18:30,217 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:18:30,217 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:18:30,218 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:18:30,218 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:18:30,219 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:18:30,219 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:18:30,219 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:18:30,220 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-18 18:18:30,222 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:18:30,223 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-18 18:18:30,231 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-18 18:18:30,231 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-18 18:18:30,233 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-18 18:18:30,234 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-18 18:18:30,248 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-18 18:18:30,251 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-18 18:18:30,258 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-18 18:18:30,258 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-18 18:18:30,259 main DEBUG createAppenders(={Console}) 2024-11-18 18:18:30,260 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 initialized 2024-11-18 18:18:30,260 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 2024-11-18 18:18:30,261 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@57bc27f5 OK. 2024-11-18 18:18:30,261 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-18 18:18:30,262 main DEBUG OutputStream closed 2024-11-18 18:18:30,262 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-18 18:18:30,262 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-18 18:18:30,263 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@57cf54e1 OK 2024-11-18 18:18:30,539 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-18 18:18:30,544 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-18 18:18:30,548 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-18 18:18:30,552 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-18 18:18:30,559 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-18 18:18:30,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-18 18:18:30,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-18 18:18:30,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-18 18:18:30,571 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-18 18:18:30,572 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-18 18:18:30,572 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-18 18:18:30,573 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-18 18:18:30,574 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-18 18:18:30,575 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-18 18:18:30,576 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-18 18:18:30,577 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-18 18:18:30,577 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-18 18:18:30,578 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-18 18:18:30,602 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18 18:18:30,602 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@1608bcbd) with optional ClassLoader: null 2024-11-18 18:18:30,603 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-18 18:18:30,604 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@1608bcbd] started OK. 2024-11-18T18:18:30,656 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-18 18:18:30,660 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-18 18:18:30,660 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18T18:18:31,490 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8 2024-11-18T18:18:31,567 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-18T18:18:31,659 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/cluster_029d3cf4-b51c-5c2b-f2f7-b81fc26ce99d, deleteOnExit=true 2024-11-18T18:18:31,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-18T18:18:31,661 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/test.cache.data in system properties and HBase conf 2024-11-18T18:18:31,662 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/hadoop.tmp.dir in system properties and HBase conf 2024-11-18T18:18:31,662 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/hadoop.log.dir in system properties and HBase conf 2024-11-18T18:18:31,663 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-18T18:18:31,664 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-18T18:18:31,664 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-18T18:18:31,968 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-18T18:18:32,257 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:18:32,263 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-18T18:18:32,273 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-18T18:18:32,273 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-18T18:18:32,274 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T18:18:32,275 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-18T18:18:32,275 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-18T18:18:32,276 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T18:18:32,277 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T18:18:32,277 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-18T18:18:32,278 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/nfs.dump.dir in system properties and HBase conf 2024-11-18T18:18:32,279 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/java.io.tmpdir in system properties and HBase conf 2024-11-18T18:18:32,279 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T18:18:32,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-18T18:18:32,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-18T18:18:34,357 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-18T18:18:34,566 INFO [Time-limited test {}] log.Log(170): Logging initialized @6570ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-18T18:18:34,754 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T18:18:34,950 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T18:18:35,056 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T18:18:35,056 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T18:18:35,058 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T18:18:35,093 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T18:18:35,099 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@24268d3a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/hadoop.log.dir/,AVAILABLE} 2024-11-18T18:18:35,103 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@549fd2eb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T18:18:35,494 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@b5692ae{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/java.io.tmpdir/jetty-localhost-46239-hadoop-hdfs-3_4_1-tests_jar-_-any-1120408948753533327/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T18:18:35,522 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1809bc91{HTTP/1.1, (http/1.1)}{localhost:46239} 2024-11-18T18:18:35,523 INFO [Time-limited test {}] server.Server(415): Started @7529ms 2024-11-18T18:18:36,304 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T18:18:36,316 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T18:18:36,322 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T18:18:36,322 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T18:18:36,324 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T18:18:36,341 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d71513b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/hadoop.log.dir/,AVAILABLE} 2024-11-18T18:18:36,342 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2be8c12a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T18:18:36,627 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1362c9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/java.io.tmpdir/jetty-localhost-46115-hadoop-hdfs-3_4_1-tests_jar-_-any-16050345910769709919/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T18:18:36,628 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b8ee7e7{HTTP/1.1, (http/1.1)}{localhost:46115} 2024-11-18T18:18:36,629 INFO [Time-limited test {}] server.Server(415): Started @8634ms 2024-11-18T18:18:36,723 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T18:18:39,194 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/cluster_029d3cf4-b51c-5c2b-f2f7-b81fc26ce99d/data/data1/current/BP-100917435-172.17.0.2-1731953913846/current, will proceed with Du for space computation calculation, 2024-11-18T18:18:39,195 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/cluster_029d3cf4-b51c-5c2b-f2f7-b81fc26ce99d/data/data2/current/BP-100917435-172.17.0.2-1731953913846/current, will proceed with Du for space computation calculation, 2024-11-18T18:18:39,307 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T18:18:39,396 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1ea749e33bb79057 with lease ID 0xa20edc1b73e07bbd: Processing first storage report for DS-544416a8-ab31-4f5b-bf7e-91811addb4c4 from datanode DatanodeRegistration(127.0.0.1:41527, datanodeUuid=bd5f880e-bbd5-4322-b0c7-35c085034d3a, infoPort=41159, infoSecurePort=0, ipcPort=41855, storageInfo=lv=-57;cid=testClusterID;nsid=1117301303;c=1731953913846) 2024-11-18T18:18:39,397 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1ea749e33bb79057 with lease ID 0xa20edc1b73e07bbd: from storage DS-544416a8-ab31-4f5b-bf7e-91811addb4c4 node DatanodeRegistration(127.0.0.1:41527, datanodeUuid=bd5f880e-bbd5-4322-b0c7-35c085034d3a, infoPort=41159, infoSecurePort=0, ipcPort=41855, storageInfo=lv=-57;cid=testClusterID;nsid=1117301303;c=1731953913846), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-18T18:18:39,398 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1ea749e33bb79057 with lease ID 0xa20edc1b73e07bbd: Processing first storage report for DS-bba7bb4b-bc11-4388-b656-8e12a96e60f9 from datanode DatanodeRegistration(127.0.0.1:41527, datanodeUuid=bd5f880e-bbd5-4322-b0c7-35c085034d3a, infoPort=41159, infoSecurePort=0, ipcPort=41855, storageInfo=lv=-57;cid=testClusterID;nsid=1117301303;c=1731953913846) 2024-11-18T18:18:39,398 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1ea749e33bb79057 with lease ID 0xa20edc1b73e07bbd: from storage DS-bba7bb4b-bc11-4388-b656-8e12a96e60f9 node DatanodeRegistration(127.0.0.1:41527, datanodeUuid=bd5f880e-bbd5-4322-b0c7-35c085034d3a, infoPort=41159, infoSecurePort=0, ipcPort=41855, storageInfo=lv=-57;cid=testClusterID;nsid=1117301303;c=1731953913846), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-18T18:18:39,485 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8 2024-11-18T18:18:39,604 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/cluster_029d3cf4-b51c-5c2b-f2f7-b81fc26ce99d/zookeeper_0, clientPort=50225, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/cluster_029d3cf4-b51c-5c2b-f2f7-b81fc26ce99d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/cluster_029d3cf4-b51c-5c2b-f2f7-b81fc26ce99d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T18:18:39,615 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50225 2024-11-18T18:18:39,630 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T18:18:39,634 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T18:18:39,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741825_1001 (size=7) 2024-11-18T18:18:40,021 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23 with version=8 2024-11-18T18:18:40,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/hbase-staging 2024-11-18T18:18:40,174 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-18T18:18:40,584 INFO [Time-limited test {}] client.ConnectionUtils(128): master/961a81da9827:0 server-side Connection retries=9 2024-11-18T18:18:40,603 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:18:40,604 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T18:18:40,611 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T18:18:40,612 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:18:40,612 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T18:18:40,869 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T18:18:40,973 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-18T18:18:40,994 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-18T18:18:40,999 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T18:18:41,039 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1588 (auto-detected) 2024-11-18T18:18:41,040 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-18T18:18:41,068 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39417 2024-11-18T18:18:41,099 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39417 connecting to ZooKeeper ensemble=127.0.0.1:50225 2024-11-18T18:18:41,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:394170x0, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T18:18:41,171 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39417-0x10107127afd0000 connected 2024-11-18T18:18:41,332 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T18:18:41,336 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T18:18:41,355 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:18:41,365 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23, hbase.cluster.distributed=false 2024-11-18T18:18:41,458 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T18:18:41,473 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39417 2024-11-18T18:18:41,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39417 2024-11-18T18:18:41,489 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39417 2024-11-18T18:18:41,506 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39417 2024-11-18T18:18:41,516 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39417 2024-11-18T18:18:41,744 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/961a81da9827:0 server-side Connection retries=9 2024-11-18T18:18:41,747 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:18:41,748 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T18:18:41,748 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T18:18:41,748 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:18:41,749 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T18:18:41,752 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T18:18:41,764 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T18:18:41,766 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38009 2024-11-18T18:18:41,769 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38009 connecting to ZooKeeper ensemble=127.0.0.1:50225 2024-11-18T18:18:41,770 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T18:18:41,776 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T18:18:41,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:380090x0, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T18:18:41,806 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:380090x0, quorum=127.0.0.1:50225, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:18:41,813 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T18:18:41,816 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38009-0x10107127afd0001 connected 2024-11-18T18:18:41,840 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T18:18:41,859 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38009-0x10107127afd0001, quorum=127.0.0.1:50225, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T18:18:41,888 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38009-0x10107127afd0001, quorum=127.0.0.1:50225, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T18:18:41,890 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38009 2024-11-18T18:18:41,893 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38009 2024-11-18T18:18:41,905 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38009 2024-11-18T18:18:41,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38009 2024-11-18T18:18:41,921 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38009 2024-11-18T18:18:41,954 DEBUG [M:0;961a81da9827:39417 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;961a81da9827:39417 2024-11-18T18:18:41,955 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/961a81da9827,39417,1731953920254 2024-11-18T18:18:41,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38009-0x10107127afd0001, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:18:41,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:18:41,970 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/961a81da9827,39417,1731953920254 2024-11-18T18:18:42,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38009-0x10107127afd0001, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T18:18:42,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38009-0x10107127afd0001, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:18:42,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:18:42,008 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T18:18:42,010 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/961a81da9827,39417,1731953920254 from backup master directory 2024-11-18T18:18:42,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/961a81da9827,39417,1731953920254 2024-11-18T18:18:42,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38009-0x10107127afd0001, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:18:42,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:18:42,014 WARN [master/961a81da9827:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T18:18:42,014 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=961a81da9827,39417,1731953920254 2024-11-18T18:18:42,018 INFO [master/961a81da9827:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-18T18:18:42,020 INFO [master/961a81da9827:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-18T18:18:42,100 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/hbase.id] with ID: f2a2c8e3-1adc-420e-aea6-91edb6c325c2 2024-11-18T18:18:42,100 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/.tmp/hbase.id 2024-11-18T18:18:42,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741826_1002 (size=42) 2024-11-18T18:18:42,534 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/.tmp/hbase.id]:[hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/hbase.id] 2024-11-18T18:18:42,617 INFO [master/961a81da9827:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T18:18:42,634 INFO [master/961a81da9827:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T18:18:42,679 INFO [master/961a81da9827:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 41ms. 2024-11-18T18:18:42,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38009-0x10107127afd0001, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:18:42,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:18:42,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741827_1003 (size=196) 2024-11-18T18:18:43,162 INFO [master/961a81da9827:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T18:18:43,167 INFO [master/961a81da9827:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T18:18:43,194 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:18:43,201 INFO [master/961a81da9827:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T18:18:43,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741828_1004 (size=1189) 2024-11-18T18:18:43,700 INFO [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/data/master/store 2024-11-18T18:18:43,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741829_1005 (size=34) 2024-11-18T18:18:44,199 INFO [master/961a81da9827:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-18T18:18:44,207 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:18:44,210 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T18:18:44,210 INFO [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:18:44,211 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:18:44,221 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T18:18:44,221 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:18:44,221 INFO [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:18:44,223 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731953924210Disabling compacts and flushes for region at 1731953924210Disabling writes for close at 1731953924221 (+11 ms)Writing region close event to WAL at 1731953924221Closed at 1731953924221 2024-11-18T18:18:44,238 WARN [master/961a81da9827:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/data/master/store/.initializing 2024-11-18T18:18:44,238 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/WALs/961a81da9827,39417,1731953920254 2024-11-18T18:18:44,255 INFO [master/961a81da9827:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T18:18:44,276 INFO [master/961a81da9827:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=961a81da9827%2C39417%2C1731953920254, suffix=, logDir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/WALs/961a81da9827,39417,1731953920254, archiveDir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/oldWALs, maxLogs=10 2024-11-18T18:18:44,302 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/WALs/961a81da9827,39417,1731953920254/961a81da9827%2C39417%2C1731953920254.1731953924281, exclude list is [], retry=0 2024-11-18T18:18:44,333 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41527,DS-544416a8-ab31-4f5b-bf7e-91811addb4c4,DISK] 2024-11-18T18:18:44,337 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-18T18:18:44,423 INFO [master/961a81da9827:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/WALs/961a81da9827,39417,1731953920254/961a81da9827%2C39417%2C1731953920254.1731953924281 2024-11-18T18:18:44,440 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41159:41159)] 2024-11-18T18:18:44,441 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:18:44,443 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:18:44,449 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:18:44,451 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:18:44,562 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:18:44,624 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T18:18:44,629 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:18:44,635 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:18:44,642 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:18:44,658 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T18:18:44,664 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:18:44,670 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:18:44,671 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:18:44,679 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T18:18:44,680 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:18:44,683 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:18:44,683 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:18:44,690 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T18:18:44,690 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:18:44,692 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:18:44,692 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:18:44,697 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:18:44,699 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:18:44,707 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:18:44,707 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:18:44,715 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T18:18:44,727 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:18:44,742 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:18:44,744 INFO [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72436042, jitterRate=0.07938113808631897}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T18:18:44,757 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731953924470Initializing all the Stores at 1731953924497 (+27 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731953924498 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731953924509 (+11 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731953924509Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731953924510 (+1 ms)Cleaning up temporary data from old regions at 1731953924707 (+197 ms)Region opened successfully at 1731953924757 (+50 ms) 2024-11-18T18:18:44,760 INFO [master/961a81da9827:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T18:18:44,844 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b3e4f70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=961a81da9827/172.17.0.2:0 2024-11-18T18:18:44,929 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T18:18:44,951 INFO [master/961a81da9827:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-18T18:18:44,952 INFO [master/961a81da9827:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-18T18:18:44,956 INFO [master/961a81da9827:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T18:18:44,959 INFO [master/961a81da9827:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-18T18:18:44,966 INFO [master/961a81da9827:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-18T18:18:44,966 INFO [master/961a81da9827:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T18:18:45,017 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T18:18:45,030 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T18:18:45,078 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T18:18:45,082 INFO [master/961a81da9827:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T18:18:45,085 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T18:18:45,155 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T18:18:45,158 INFO [master/961a81da9827:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T18:18:45,163 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T18:18:45,217 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T18:18:45,220 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T18:18:45,270 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T18:18:45,293 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T18:18:45,295 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T18:18:45,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T18:18:45,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38009-0x10107127afd0001, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T18:18:45,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38009-0x10107127afd0001, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:18:45,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:18:45,305 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=961a81da9827,39417,1731953920254, sessionid=0x10107127afd0000, setting cluster-up flag (Was=false) 2024-11-18T18:18:45,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38009-0x10107127afd0001, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:18:45,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:18:45,340 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T18:18:45,342 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=961a81da9827,39417,1731953920254 2024-11-18T18:18:45,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:18:45,371 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38009-0x10107127afd0001, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:18:45,416 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T18:18:45,420 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=961a81da9827,39417,1731953920254 2024-11-18T18:18:45,430 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T18:18:45,451 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(746): ClusterId : f2a2c8e3-1adc-420e-aea6-91edb6c325c2 2024-11-18T18:18:45,454 DEBUG [RS:0;961a81da9827:38009 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T18:18:45,466 DEBUG [RS:0;961a81da9827:38009 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T18:18:45,466 DEBUG [RS:0;961a81da9827:38009 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T18:18:45,485 DEBUG [RS:0;961a81da9827:38009 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T18:18:45,489 DEBUG [RS:0;961a81da9827:38009 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4acc6652, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=961a81da9827/172.17.0.2:0 2024-11-18T18:18:45,530 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;961a81da9827:38009 2024-11-18T18:18:45,534 INFO [RS:0;961a81da9827:38009 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T18:18:45,534 INFO [RS:0;961a81da9827:38009 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T18:18:45,534 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T18:18:45,540 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(2659): reportForDuty to master=961a81da9827,39417,1731953920254 with port=38009, startcode=1731953921685 2024-11-18T18:18:45,550 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T18:18:45,558 DEBUG [RS:0;961a81da9827:38009 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T18:18:45,565 INFO [master/961a81da9827:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T18:18:45,577 INFO [master/961a81da9827:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T18:18:45,589 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 961a81da9827,39417,1731953920254 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T18:18:45,607 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/961a81da9827:0, corePoolSize=5, maxPoolSize=5 2024-11-18T18:18:45,607 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/961a81da9827:0, corePoolSize=5, maxPoolSize=5 2024-11-18T18:18:45,608 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/961a81da9827:0, corePoolSize=5, maxPoolSize=5 2024-11-18T18:18:45,608 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/961a81da9827:0, corePoolSize=5, maxPoolSize=5 2024-11-18T18:18:45,608 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/961a81da9827:0, corePoolSize=10, maxPoolSize=10 2024-11-18T18:18:45,609 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:18:45,609 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/961a81da9827:0, corePoolSize=2, maxPoolSize=2 2024-11-18T18:18:45,609 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:18:45,627 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55357, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T18:18:45,634 INFO [master/961a81da9827:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731953955634 2024-11-18T18:18:45,635 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39417 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-18T18:18:45,636 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T18:18:45,637 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T18:18:45,658 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T18:18:45,659 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T18:18:45,682 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T18:18:45,682 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T18:18:45,683 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T18:18:45,683 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:18:45,683 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T18:18:45,683 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T18:18:45,685 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T18:18:45,711 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T18:18:45,713 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T18:18:45,715 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T18:18:45,730 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T18:18:45,731 INFO [master/961a81da9827:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T18:18:45,734 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-18T18:18:45,735 WARN [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 30000 ms and then retrying. 2024-11-18T18:18:45,735 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/961a81da9827:0:becomeActiveMaster-HFileCleaner.large.0-1731953925732,5,FailOnTimeoutGroup] 2024-11-18T18:18:45,737 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/961a81da9827:0:becomeActiveMaster-HFileCleaner.small.0-1731953925735,5,FailOnTimeoutGroup] 2024-11-18T18:18:45,737 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T18:18:45,738 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T18:18:45,739 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T18:18:45,740 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T18:18:45,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741831_1007 (size=1321) 2024-11-18T18:18:45,773 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T18:18:45,774 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23 2024-11-18T18:18:45,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741832_1008 (size=32) 2024-11-18T18:18:45,839 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:18:45,850 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T18:18:45,856 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T18:18:45,857 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:18:45,878 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:18:45,879 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T18:18:45,899 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T18:18:45,899 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:18:45,901 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:18:45,901 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T18:18:45,913 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T18:18:45,916 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:18:45,929 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:18:45,934 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T18:18:45,957 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T18:18:45,957 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:18:45,966 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:18:45,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T18:18:45,982 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740 2024-11-18T18:18:45,986 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740 2024-11-18T18:18:46,019 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T18:18:46,019 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T18:18:46,026 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T18:18:46,040 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T18:18:46,059 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:18:46,062 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68955209, jitterRate=0.02751268446445465}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T18:18:46,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731953925839Initializing all the Stores at 1731953925847 (+8 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731953925847Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731953925849 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731953925852 (+3 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731953925852Cleaning up temporary data from old regions at 1731953926019 (+167 ms)Region opened successfully at 1731953926066 (+47 ms) 2024-11-18T18:18:46,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T18:18:46,066 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T18:18:46,067 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T18:18:46,067 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T18:18:46,067 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T18:18:46,072 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T18:18:46,073 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731953926066Disabling compacts and flushes for region at 1731953926066Disabling writes for close at 1731953926067 (+1 ms)Writing region close event to WAL at 1731953926072 (+5 ms)Closed at 1731953926072 2024-11-18T18:18:46,081 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T18:18:46,081 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T18:18:46,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T18:18:46,109 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T18:18:46,114 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T18:18:46,265 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:18:47,266 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:18:48,267 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:18:49,277 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:18:50,281 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:18:50,960 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-18T18:18:50,960 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-18T18:18:51,282 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:18:52,283 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:18:53,284 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:18:54,285 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:18:55,286 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:18:56,287 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:18:57,287 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:18:58,288 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:18:59,289 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:00,290 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:01,291 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:02,291 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:03,292 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:04,293 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:05,293 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:06,294 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:07,295 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:08,296 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:09,297 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:09,490 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T18:19:10,297 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:11,298 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:12,308 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:13,308 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:14,309 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:15,310 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:15,737 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(2659): reportForDuty to master=961a81da9827,39417,1731953920254 with port=38009, startcode=1731953921685 2024-11-18T18:19:15,742 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39417 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 961a81da9827,38009,1731953921685 2024-11-18T18:19:15,746 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39417 {}] master.ServerManager(517): Registering regionserver=961a81da9827,38009,1731953921685 2024-11-18T18:19:15,766 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23 2024-11-18T18:19:15,766 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42805 2024-11-18T18:19:15,766 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T18:19:15,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T18:19:15,782 DEBUG [RS:0;961a81da9827:38009 {}] zookeeper.ZKUtil(111): regionserver:38009-0x10107127afd0001, quorum=127.0.0.1:50225, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/961a81da9827,38009,1731953921685 2024-11-18T18:19:15,782 WARN [RS:0;961a81da9827:38009 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T18:19:15,782 INFO [RS:0;961a81da9827:38009 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T18:19:15,782 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/WALs/961a81da9827,38009,1731953921685 2024-11-18T18:19:15,789 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [961a81da9827,38009,1731953921685] 2024-11-18T18:19:15,823 INFO [RS:0;961a81da9827:38009 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T18:19:15,841 INFO [RS:0;961a81da9827:38009 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T18:19:15,849 INFO [RS:0;961a81da9827:38009 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T18:19:15,850 INFO [RS:0;961a81da9827:38009 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:19:15,854 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T18:19:15,861 INFO [RS:0;961a81da9827:38009 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T18:19:15,863 INFO [RS:0;961a81da9827:38009 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T18:19:15,863 DEBUG [RS:0;961a81da9827:38009 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:19:15,864 DEBUG [RS:0;961a81da9827:38009 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:19:15,864 DEBUG [RS:0;961a81da9827:38009 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:19:15,864 DEBUG [RS:0;961a81da9827:38009 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:19:15,864 DEBUG [RS:0;961a81da9827:38009 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:19:15,864 DEBUG [RS:0;961a81da9827:38009 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/961a81da9827:0, corePoolSize=2, maxPoolSize=2 2024-11-18T18:19:15,864 DEBUG [RS:0;961a81da9827:38009 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:19:15,865 DEBUG [RS:0;961a81da9827:38009 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:19:15,865 DEBUG [RS:0;961a81da9827:38009 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:19:15,865 DEBUG [RS:0;961a81da9827:38009 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:19:15,865 DEBUG [RS:0;961a81da9827:38009 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:19:15,865 DEBUG [RS:0;961a81da9827:38009 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/961a81da9827:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:19:15,865 DEBUG [RS:0;961a81da9827:38009 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/961a81da9827:0, corePoolSize=3, maxPoolSize=3 2024-11-18T18:19:15,865 DEBUG [RS:0;961a81da9827:38009 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/961a81da9827:0, corePoolSize=3, maxPoolSize=3 2024-11-18T18:19:15,871 INFO [RS:0;961a81da9827:38009 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T18:19:15,872 INFO [RS:0;961a81da9827:38009 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T18:19:15,872 INFO [RS:0;961a81da9827:38009 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:19:15,872 INFO [RS:0;961a81da9827:38009 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T18:19:15,872 INFO [RS:0;961a81da9827:38009 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T18:19:15,872 INFO [RS:0;961a81da9827:38009 {}] hbase.ChoreService(168): Chore ScheduledChore name=961a81da9827,38009,1731953921685-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T18:19:15,905 INFO [RS:0;961a81da9827:38009 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T18:19:15,908 INFO [RS:0;961a81da9827:38009 {}] hbase.ChoreService(168): Chore ScheduledChore name=961a81da9827,38009,1731953921685-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:19:15,909 INFO [RS:0;961a81da9827:38009 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:19:15,909 INFO [RS:0;961a81da9827:38009 {}] regionserver.Replication(171): 961a81da9827,38009,1731953921685 started 2024-11-18T18:19:15,933 INFO [RS:0;961a81da9827:38009 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:19:15,934 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1482): Serving as 961a81da9827,38009,1731953921685, RpcServer on 961a81da9827/172.17.0.2:38009, sessionid=0x10107127afd0001 2024-11-18T18:19:15,935 DEBUG [RS:0;961a81da9827:38009 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T18:19:15,935 DEBUG [RS:0;961a81da9827:38009 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 961a81da9827,38009,1731953921685 2024-11-18T18:19:15,936 DEBUG [RS:0;961a81da9827:38009 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '961a81da9827,38009,1731953921685' 2024-11-18T18:19:15,936 DEBUG [RS:0;961a81da9827:38009 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T18:19:15,937 DEBUG [RS:0;961a81da9827:38009 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T18:19:15,938 DEBUG [RS:0;961a81da9827:38009 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T18:19:15,939 DEBUG [RS:0;961a81da9827:38009 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T18:19:15,939 DEBUG [RS:0;961a81da9827:38009 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 961a81da9827,38009,1731953921685 2024-11-18T18:19:15,939 DEBUG [RS:0;961a81da9827:38009 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '961a81da9827,38009,1731953921685' 2024-11-18T18:19:15,939 DEBUG [RS:0;961a81da9827:38009 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T18:19:15,940 DEBUG [RS:0;961a81da9827:38009 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T18:19:15,941 DEBUG [RS:0;961a81da9827:38009 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T18:19:15,941 INFO [RS:0;961a81da9827:38009 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T18:19:15,942 INFO [RS:0;961a81da9827:38009 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T18:19:16,311 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:17,320 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:18,321 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:19,322 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:20,322 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:21,323 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:22,324 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:23,328 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:24,331 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:25,332 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:26,333 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:27,336 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:28,337 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:29,338 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:30,339 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:31,339 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:32,340 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:33,341 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:34,342 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:35,343 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:36,344 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:37,345 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:38,346 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:39,347 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:39,491 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T18:19:40,349 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:41,351 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:42,351 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:43,361 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:44,362 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:45,363 WARN [961a81da9827:39417 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:19:45,948 INFO [RS:0;961a81da9827:38009 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T18:19:45,953 INFO [RS:0;961a81da9827:38009 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=961a81da9827%2C38009%2C1731953921685, suffix=, logDir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/WALs/961a81da9827,38009,1731953921685, archiveDir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/oldWALs, maxLogs=32 2024-11-18T18:19:45,971 DEBUG [RS:0;961a81da9827:38009 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/WALs/961a81da9827,38009,1731953921685/961a81da9827%2C38009%2C1731953921685.1731953985956, exclude list is [], retry=0 2024-11-18T18:19:45,976 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41527,DS-544416a8-ab31-4f5b-bf7e-91811addb4c4,DISK] 2024-11-18T18:19:45,980 INFO [RS:0;961a81da9827:38009 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/WALs/961a81da9827,38009,1731953921685/961a81da9827%2C38009%2C1731953921685.1731953985956 2024-11-18T18:19:45,980 DEBUG [RS:0;961a81da9827:38009 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41159:41159)] 2024-11-18T18:19:46,116 DEBUG [961a81da9827:39417 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-18T18:19:46,132 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:19:46,140 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 961a81da9827,38009,1731953921685, state=OPENING 2024-11-18T18:19:46,150 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T18:19:46,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:19:46,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38009-0x10107127afd0001, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:19:46,159 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:19:46,159 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:19:46,163 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T18:19:46,166 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=961a81da9827,38009,1731953921685}] 2024-11-18T18:19:46,354 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T18:19:46,358 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39707, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T18:19:46,373 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T18:19:46,374 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T18:19:46,375 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T18:19:46,380 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=961a81da9827%2C38009%2C1731953921685.meta, suffix=.meta, logDir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/WALs/961a81da9827,38009,1731953921685, archiveDir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/oldWALs, maxLogs=32 2024-11-18T18:19:46,406 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/WALs/961a81da9827,38009,1731953921685/961a81da9827%2C38009%2C1731953921685.meta.1731953986383.meta, exclude list is [], retry=0 2024-11-18T18:19:46,416 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41527,DS-544416a8-ab31-4f5b-bf7e-91811addb4c4,DISK] 2024-11-18T18:19:46,430 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/WALs/961a81da9827,38009,1731953921685/961a81da9827%2C38009%2C1731953921685.meta.1731953986383.meta 2024-11-18T18:19:46,431 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41159:41159)] 2024-11-18T18:19:46,432 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:19:46,434 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T18:19:46,438 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T18:19:46,444 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T18:19:46,449 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T18:19:46,450 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:19:46,450 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T18:19:46,451 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T18:19:46,457 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T18:19:46,459 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T18:19:46,459 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:19:46,460 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:19:46,460 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T18:19:46,462 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T18:19:46,462 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:19:46,463 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:19:46,463 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T18:19:46,465 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T18:19:46,465 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:19:46,466 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:19:46,466 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T18:19:46,468 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T18:19:46,468 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:19:46,469 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:19:46,469 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T18:19:46,470 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740 2024-11-18T18:19:46,473 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740 2024-11-18T18:19:46,476 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T18:19:46,476 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T18:19:46,478 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T18:19:46,481 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T18:19:46,482 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69311247, jitterRate=0.03281806409358978}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T18:19:46,483 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T18:19:46,484 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731953986451Writing region info on filesystem at 1731953986452 (+1 ms)Initializing all the Stores at 1731953986455 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731953986455Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731953986456 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731953986456Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731953986457 (+1 ms)Cleaning up temporary data from old regions at 1731953986477 (+20 ms)Running coprocessor post-open hooks at 1731953986483 (+6 ms)Region opened successfully at 1731953986484 (+1 ms) 2024-11-18T18:19:46,493 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731953986341 2024-11-18T18:19:46,505 DEBUG [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T18:19:46,506 INFO [RS_OPEN_META-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T18:19:46,508 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:19:46,511 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 961a81da9827,38009,1731953921685, state=OPEN 2024-11-18T18:19:46,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38009-0x10107127afd0001, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:19:46,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:19:46,515 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:19:46,515 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:19:46,515 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=961a81da9827,38009,1731953921685 2024-11-18T18:19:46,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T18:19:46,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=961a81da9827,38009,1731953921685 in 350 msec 2024-11-18T18:19:46,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T18:19:46,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1 mins, 0.429 sec 2024-11-18T18:19:46,532 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T18:19:46,532 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T18:19:46,557 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:19:46,559 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,38009,1731953921685, seqNum=-1] 2024-11-18T18:19:46,587 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T18:19:46,590 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41173, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T18:19:46,614 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1 mins, 1.136 sec 2024-11-18T18:19:46,614 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731953986614, completionTime=-1 2024-11-18T18:19:46,618 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-18T18:19:46,618 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T18:19:46,654 INFO [master/961a81da9827:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-18T18:19:46,654 INFO [master/961a81da9827:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731954046654 2024-11-18T18:19:46,655 INFO [master/961a81da9827:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731954106654 2024-11-18T18:19:46,655 INFO [master/961a81da9827:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 36 msec 2024-11-18T18:19:46,658 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=961a81da9827,39417,1731953920254-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:19:46,658 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=961a81da9827,39417,1731953920254-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:19:46,658 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=961a81da9827,39417,1731953920254-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:19:46,660 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-961a81da9827:39417, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:19:46,661 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T18:19:46,661 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T18:19:46,668 DEBUG [master/961a81da9827:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T18:19:46,696 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 64.681sec 2024-11-18T18:19:46,698 INFO [master/961a81da9827:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T18:19:46,700 INFO [master/961a81da9827:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T18:19:46,701 INFO [master/961a81da9827:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T18:19:46,702 INFO [master/961a81da9827:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T18:19:46,702 INFO [master/961a81da9827:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T18:19:46,703 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=961a81da9827,39417,1731953920254-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T18:19:46,704 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=961a81da9827,39417,1731953920254-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T18:19:46,714 DEBUG [master/961a81da9827:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T18:19:46,715 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T18:19:46,716 INFO [master/961a81da9827:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=961a81da9827,39417,1731953920254-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:19:46,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@789d4313, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T18:19:46,785 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-18T18:19:46,785 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-18T18:19:46,790 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 961a81da9827,39417,-1 for getting cluster id 2024-11-18T18:19:46,794 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T18:19:46,803 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f2a2c8e3-1adc-420e-aea6-91edb6c325c2' 2024-11-18T18:19:46,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T18:19:46,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f2a2c8e3-1adc-420e-aea6-91edb6c325c2" 2024-11-18T18:19:46,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a5f5f6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T18:19:46,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [961a81da9827,39417,-1] 2024-11-18T18:19:46,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T18:19:46,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:19:46,817 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T18:19:46,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f230cde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T18:19:46,821 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:19:46,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,38009,1731953921685, seqNum=-1] 2024-11-18T18:19:46,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T18:19:46,832 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T18:19:46,894 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=961a81da9827,39417,1731953920254 2024-11-18T18:19:46,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ea61ffb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T18:19:46,898 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 961a81da9827,39417,-1 for getting cluster id 2024-11-18T18:19:46,898 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T18:19:46,900 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f2a2c8e3-1adc-420e-aea6-91edb6c325c2' 2024-11-18T18:19:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T18:19:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f2a2c8e3-1adc-420e-aea6-91edb6c325c2" 2024-11-18T18:19:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e8a2617, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T18:19:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [961a81da9827,39417,-1] 2024-11-18T18:19:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T18:19:46,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:19:46,908 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T18:19:46,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69e4f460, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T18:19:46,937 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=208, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=741, ProcessCount=11, AvailableMemoryMB=1551 2024-11-18T18:19:46,946 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T18:19:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 961a81da9827,39417,1731953920254 2024-11-18T18:19:46,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1ee78c4f 2024-11-18T18:19:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T18:19:46,966 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T18:19:46,974 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T18:19:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-18T18:19:46,987 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T18:19:46,989 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-18T18:19:46,990 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:19:46,992 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T18:19:46,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T18:19:47,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741835_1011 (size=428) 2024-11-18T18:19:47,411 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 19f5e2dceb3dffa2d39e146c4692d623, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23 2024-11-18T18:19:47,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741836_1012 (size=79) 2024-11-18T18:19:47,424 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:19:47,425 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 19f5e2dceb3dffa2d39e146c4692d623, disabling compactions & flushes 2024-11-18T18:19:47,425 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623. 2024-11-18T18:19:47,425 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623. 2024-11-18T18:19:47,425 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623. after waiting 0 ms 2024-11-18T18:19:47,425 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623. 2024-11-18T18:19:47,425 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623. 2024-11-18T18:19:47,425 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 19f5e2dceb3dffa2d39e146c4692d623: Waiting for close lock at 1731953987424Disabling compacts and flushes for region at 1731953987424Disabling writes for close at 1731953987425 (+1 ms)Writing region close event to WAL at 1731953987425Closed at 1731953987425 2024-11-18T18:19:47,428 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T18:19:47,433 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731953987428"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731953987428"}]},"ts":"1731953987428"} 2024-11-18T18:19:47,439 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T18:19:47,441 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T18:19:47,445 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731953987442"}]},"ts":"1731953987442"} 2024-11-18T18:19:47,450 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-18T18:19:47,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=19f5e2dceb3dffa2d39e146c4692d623, ASSIGN}] 2024-11-18T18:19:47,455 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=19f5e2dceb3dffa2d39e146c4692d623, ASSIGN 2024-11-18T18:19:47,457 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=19f5e2dceb3dffa2d39e146c4692d623, ASSIGN; state=OFFLINE, location=961a81da9827,38009,1731953921685; forceNewPlan=false, retain=false 2024-11-18T18:19:47,609 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=19f5e2dceb3dffa2d39e146c4692d623, regionState=OPENING, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:19:47,615 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=19f5e2dceb3dffa2d39e146c4692d623, ASSIGN because future has completed 2024-11-18T18:19:47,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 19f5e2dceb3dffa2d39e146c4692d623, server=961a81da9827,38009,1731953921685}] 2024-11-18T18:19:47,778 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623. 2024-11-18T18:19:47,778 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 19f5e2dceb3dffa2d39e146c4692d623, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623.', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:19:47,779 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 19f5e2dceb3dffa2d39e146c4692d623 2024-11-18T18:19:47,779 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:19:47,779 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 19f5e2dceb3dffa2d39e146c4692d623 2024-11-18T18:19:47,779 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 19f5e2dceb3dffa2d39e146c4692d623 2024-11-18T18:19:47,782 INFO [StoreOpener-19f5e2dceb3dffa2d39e146c4692d623-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 19f5e2dceb3dffa2d39e146c4692d623 2024-11-18T18:19:47,784 INFO [StoreOpener-19f5e2dceb3dffa2d39e146c4692d623-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 19f5e2dceb3dffa2d39e146c4692d623 columnFamilyName testFamily 2024-11-18T18:19:47,784 DEBUG [StoreOpener-19f5e2dceb3dffa2d39e146c4692d623-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:19:47,786 INFO [StoreOpener-19f5e2dceb3dffa2d39e146c4692d623-1 {}] regionserver.HStore(327): Store=19f5e2dceb3dffa2d39e146c4692d623/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:19:47,786 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 19f5e2dceb3dffa2d39e146c4692d623 2024-11-18T18:19:47,787 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testRetryOutOfOrderScannerNextExceptionAsync/19f5e2dceb3dffa2d39e146c4692d623 2024-11-18T18:19:47,788 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testRetryOutOfOrderScannerNextExceptionAsync/19f5e2dceb3dffa2d39e146c4692d623 2024-11-18T18:19:47,789 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 19f5e2dceb3dffa2d39e146c4692d623 2024-11-18T18:19:47,789 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 19f5e2dceb3dffa2d39e146c4692d623 2024-11-18T18:19:47,793 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 19f5e2dceb3dffa2d39e146c4692d623 2024-11-18T18:19:47,797 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testRetryOutOfOrderScannerNextExceptionAsync/19f5e2dceb3dffa2d39e146c4692d623/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:19:47,798 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 19f5e2dceb3dffa2d39e146c4692d623; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64946476, jitterRate=-0.032222092151641846}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T18:19:47,798 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 19f5e2dceb3dffa2d39e146c4692d623 2024-11-18T18:19:47,799 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 19f5e2dceb3dffa2d39e146c4692d623: Running coprocessor pre-open hook at 1731953987779Writing region info on filesystem at 1731953987780 (+1 ms)Initializing all the Stores at 1731953987781 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731953987781Cleaning up temporary data from old regions at 1731953987789 (+8 ms)Running coprocessor post-open hooks at 1731953987798 (+9 ms)Region opened successfully at 1731953987799 (+1 ms) 2024-11-18T18:19:47,802 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623., pid=6, masterSystemTime=1731953987771 2024-11-18T18:19:47,806 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623. 2024-11-18T18:19:47,806 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623. 2024-11-18T18:19:47,807 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=19f5e2dceb3dffa2d39e146c4692d623, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:19:47,811 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 19f5e2dceb3dffa2d39e146c4692d623, server=961a81da9827,38009,1731953921685 because future has completed 2024-11-18T18:19:47,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-18T18:19:47,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 19f5e2dceb3dffa2d39e146c4692d623, server=961a81da9827,38009,1731953921685 in 197 msec 2024-11-18T18:19:47,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-18T18:19:47,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=19f5e2dceb3dffa2d39e146c4692d623, ASSIGN in 364 msec 2024-11-18T18:19:47,822 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T18:19:47,823 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731953987822"}]},"ts":"1731953987822"} 2024-11-18T18:19:47,826 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-18T18:19:47,828 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T18:19:47,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 850 msec 2024-11-18T18:19:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T18:19:48,019 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-18T18:19:48,019 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-18T18:19:48,020 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:19:48,026 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-18T18:19:48,027 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:19:48,027 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-18T18:19:48,030 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:19:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=961a81da9827,38009,1731953921685, seqNum=-1] 2024-11-18T18:19:48,033 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T18:19:48,038 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T18:19:48,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623., hostname=961a81da9827,38009,1731953921685, seqNum=2] 2024-11-18T18:19:48,060 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-18T18:19:48,066 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-18T18:19:48,071 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-18T18:19:48,076 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-18T18:19:48,078 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-18T18:19:48,079 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=6208426143205294088 2024-11-18T18:19:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:42496 deadline: 1731953991077, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-18T18:19:48,087 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-18T18:19:48,089 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-18T18:19:48,091 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-18T18:19:48,092 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-18T18:19:48,094 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=6208426143205294090 2024-11-18T18:19:48,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:42496 deadline: 1731953991092, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-18T18:19:48,098 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-18T18:19:48,099 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=6208426143205294091 2024-11-18T18:19:48,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:42496 deadline: 1731953991098, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-18T18:19:48,104 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-18T18:19:48,105 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=6208426143205294092 2024-11-18T18:19:48,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:42496 deadline: 1731953991103, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-18T18:19:48,109 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-18T18:19:48,122 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=215 (was 208) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-838902606_22 at /127.0.0.1:48946 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/961a81da9827:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=741 (was 741), ProcessCount=11 (was 11), AvailableMemoryMB=1548 (was 1551) 2024-11-18T18:19:48,135 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=215, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=741, ProcessCount=11, AvailableMemoryMB=1547 2024-11-18T18:19:48,139 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39417 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T18:19:48,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39417 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-18T18:19:48,144 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T18:19:48,145 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:19:48,145 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39417 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-18T18:19:48,147 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T18:19:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T18:19:48,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741837_1013 (size=414) 2024-11-18T18:19:48,159 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ca5265d377131ea04085fefa66db9922, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23 2024-11-18T18:19:48,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741838_1014 (size=65) 2024-11-18T18:19:48,568 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:19:48,569 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing ca5265d377131ea04085fefa66db9922, disabling compactions & flushes 2024-11-18T18:19:48,569 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922. 2024-11-18T18:19:48,569 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922. 2024-11-18T18:19:48,569 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922. after waiting 0 ms 2024-11-18T18:19:48,569 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922. 2024-11-18T18:19:48,569 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922. 2024-11-18T18:19:48,569 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for ca5265d377131ea04085fefa66db9922: Waiting for close lock at 1731953988569Disabling compacts and flushes for region at 1731953988569Disabling writes for close at 1731953988569Writing region close event to WAL at 1731953988569Closed at 1731953988569 2024-11-18T18:19:48,572 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T18:19:48,572 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1731953988572"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731953988572"}]},"ts":"1731953988572"} 2024-11-18T18:19:48,577 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T18:19:48,579 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T18:19:48,579 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731953988579"}]},"ts":"1731953988579"} 2024-11-18T18:19:48,583 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-18T18:19:48,584 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ca5265d377131ea04085fefa66db9922, ASSIGN}] 2024-11-18T18:19:48,586 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ca5265d377131ea04085fefa66db9922, ASSIGN 2024-11-18T18:19:48,587 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ca5265d377131ea04085fefa66db9922, ASSIGN; state=OFFLINE, location=961a81da9827,38009,1731953921685; forceNewPlan=false, retain=false 2024-11-18T18:19:48,740 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ca5265d377131ea04085fefa66db9922, regionState=OPENING, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:19:48,745 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ca5265d377131ea04085fefa66db9922, ASSIGN because future has completed 2024-11-18T18:19:48,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca5265d377131ea04085fefa66db9922, server=961a81da9827,38009,1731953921685}] 2024-11-18T18:19:48,909 INFO [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922. 2024-11-18T18:19:48,909 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => ca5265d377131ea04085fefa66db9922, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922.', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:19:48,909 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner ca5265d377131ea04085fefa66db9922 2024-11-18T18:19:48,909 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:19:48,910 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for ca5265d377131ea04085fefa66db9922 2024-11-18T18:19:48,910 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for ca5265d377131ea04085fefa66db9922 2024-11-18T18:19:48,912 INFO [StoreOpener-ca5265d377131ea04085fefa66db9922-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ca5265d377131ea04085fefa66db9922 2024-11-18T18:19:48,915 INFO [StoreOpener-ca5265d377131ea04085fefa66db9922-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca5265d377131ea04085fefa66db9922 columnFamilyName testFamily 2024-11-18T18:19:48,915 DEBUG [StoreOpener-ca5265d377131ea04085fefa66db9922-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:19:48,916 INFO [StoreOpener-ca5265d377131ea04085fefa66db9922-1 {}] regionserver.HStore(327): Store=ca5265d377131ea04085fefa66db9922/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:19:48,916 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for ca5265d377131ea04085fefa66db9922 2024-11-18T18:19:48,917 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnOpenScanner/ca5265d377131ea04085fefa66db9922 2024-11-18T18:19:48,918 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnOpenScanner/ca5265d377131ea04085fefa66db9922 2024-11-18T18:19:48,919 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for ca5265d377131ea04085fefa66db9922 2024-11-18T18:19:48,919 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for ca5265d377131ea04085fefa66db9922 2024-11-18T18:19:48,922 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for ca5265d377131ea04085fefa66db9922 2024-11-18T18:19:48,925 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnOpenScanner/ca5265d377131ea04085fefa66db9922/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:19:48,926 INFO [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened ca5265d377131ea04085fefa66db9922; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71974229, jitterRate=0.0724995881319046}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T18:19:48,926 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ca5265d377131ea04085fefa66db9922 2024-11-18T18:19:48,928 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for ca5265d377131ea04085fefa66db9922: Running coprocessor pre-open hook at 1731953988910Writing region info on filesystem at 1731953988910Initializing all the Stores at 1731953988911 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731953988912 (+1 ms)Cleaning up temporary data from old regions at 1731953988919 (+7 ms)Running coprocessor post-open hooks at 1731953988926 (+7 ms)Region opened successfully at 1731953988928 (+2 ms) 2024-11-18T18:19:48,930 INFO [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922., pid=9, masterSystemTime=1731953988902 2024-11-18T18:19:48,934 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922. 2024-11-18T18:19:48,935 INFO [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922. 2024-11-18T18:19:48,936 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ca5265d377131ea04085fefa66db9922, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:19:48,940 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca5265d377131ea04085fefa66db9922, server=961a81da9827,38009,1731953921685 because future has completed 2024-11-18T18:19:48,950 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-18T18:19:48,950 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure ca5265d377131ea04085fefa66db9922, server=961a81da9827,38009,1731953921685 in 198 msec 2024-11-18T18:19:48,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-18T18:19:48,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=ca5265d377131ea04085fefa66db9922, ASSIGN in 367 msec 2024-11-18T18:19:48,956 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T18:19:48,957 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731953988956"}]},"ts":"1731953988956"} 2024-11-18T18:19:48,960 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-18T18:19:48,961 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T18:19:48,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 821 msec 2024-11-18T18:19:49,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T18:19:49,158 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-18T18:19:49,158 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-18T18:19:49,159 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:19:49,166 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-18T18:19:49,166 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:19:49,166 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-18T18:19:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922., hostname=961a81da9827,38009,1731953921685, seqNum=2] 2024-11-18T18:19:49,190 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-18T18:19:49,191 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-18T18:19:49,202 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-18T18:19:49,205 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38009 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-18T18:19:50,959 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-18T18:19:50,960 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-18T18:19:50,962 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T18:19:50,963 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-18T18:19:50,963 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-18T18:19:50,963 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-18T18:19:50,968 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-18T18:19:50,968 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-18T18:19:52,002 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T18:19:52,039 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-18T18:19:52,041 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-18T18:19:52,041 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-18T18:19:55,227 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922., hostname=961a81da9827,38009,1731953921685, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922., hostname=961a81da9827,38009,1731953921685, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=961a81da9827:38009 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6015ms, rpcTimeout=5999ms 2024-11-18T18:19:55,230 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922., hostname=961a81da9827,38009,1731953921685, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6015ms, rpcTimeout=5999ms 2024-11-18T18:19:55,230 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922., hostname=961a81da9827,38009,1731953921685, seqNum=2 from cache 2024-11-18T18:19:55,253 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=218 (was 215) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=690 (was 741), ProcessCount=11 (was 11), AvailableMemoryMB=1834 (was 1547) - AvailableMemoryMB LEAK? - 2024-11-18T18:19:55,265 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=218, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=690, ProcessCount=11, AvailableMemoryMB=1832 2024-11-18T18:19:55,268 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39417 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T18:19:55,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39417 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-18T18:19:55,274 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T18:19:55,274 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:19:55,276 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39417 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-18T18:19:55,277 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T18:19:55,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-18T18:19:55,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741839_1015 (size=412) 2024-11-18T18:19:55,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-11-18T18:19:55,724 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1ab6a75433d4cdfb37b6731cf70854ba, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23 2024-11-18T18:19:55,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741840_1016 (size=63) 2024-11-18T18:19:55,757 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:19:55,757 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 1ab6a75433d4cdfb37b6731cf70854ba, disabling compactions & flushes 2024-11-18T18:19:55,758 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba. 2024-11-18T18:19:55,758 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba. 2024-11-18T18:19:55,758 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba. after waiting 0 ms 2024-11-18T18:19:55,758 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba. 2024-11-18T18:19:55,758 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba. 2024-11-18T18:19:55,758 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1ab6a75433d4cdfb37b6731cf70854ba: Waiting for close lock at 1731953995757Disabling compacts and flushes for region at 1731953995757Disabling writes for close at 1731953995758 (+1 ms)Writing region close event to WAL at 1731953995758Closed at 1731953995758 2024-11-18T18:19:55,760 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T18:19:55,761 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1731953995761"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731953995761"}]},"ts":"1731953995761"} 2024-11-18T18:19:55,775 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T18:19:55,780 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T18:19:55,780 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731953995780"}]},"ts":"1731953995780"} 2024-11-18T18:19:55,788 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-18T18:19:55,789 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=1ab6a75433d4cdfb37b6731cf70854ba, ASSIGN}] 2024-11-18T18:19:55,793 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=1ab6a75433d4cdfb37b6731cf70854ba, ASSIGN 2024-11-18T18:19:55,798 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=1ab6a75433d4cdfb37b6731cf70854ba, ASSIGN; state=OFFLINE, location=961a81da9827,38009,1731953921685; forceNewPlan=false, retain=false 2024-11-18T18:19:55,950 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=1ab6a75433d4cdfb37b6731cf70854ba, regionState=OPENING, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:19:55,957 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=1ab6a75433d4cdfb37b6731cf70854ba, ASSIGN because future has completed 2024-11-18T18:19:55,958 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ab6a75433d4cdfb37b6731cf70854ba, server=961a81da9827,38009,1731953921685}] 2024-11-18T18:19:56,128 INFO [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba. 2024-11-18T18:19:56,128 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 1ab6a75433d4cdfb37b6731cf70854ba, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba.', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:19:56,129 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 1ab6a75433d4cdfb37b6731cf70854ba 2024-11-18T18:19:56,129 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:19:56,129 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 1ab6a75433d4cdfb37b6731cf70854ba 2024-11-18T18:19:56,130 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 1ab6a75433d4cdfb37b6731cf70854ba 2024-11-18T18:19:56,142 INFO [StoreOpener-1ab6a75433d4cdfb37b6731cf70854ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1ab6a75433d4cdfb37b6731cf70854ba 2024-11-18T18:19:56,152 INFO [StoreOpener-1ab6a75433d4cdfb37b6731cf70854ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ab6a75433d4cdfb37b6731cf70854ba columnFamilyName testFamily 2024-11-18T18:19:56,153 DEBUG [StoreOpener-1ab6a75433d4cdfb37b6731cf70854ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:19:56,155 INFO [StoreOpener-1ab6a75433d4cdfb37b6731cf70854ba-1 {}] regionserver.HStore(327): Store=1ab6a75433d4cdfb37b6731cf70854ba/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:19:56,155 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 1ab6a75433d4cdfb37b6731cf70854ba 2024-11-18T18:19:56,159 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnNextAsync/1ab6a75433d4cdfb37b6731cf70854ba 2024-11-18T18:19:56,162 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnNextAsync/1ab6a75433d4cdfb37b6731cf70854ba 2024-11-18T18:19:56,165 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 1ab6a75433d4cdfb37b6731cf70854ba 2024-11-18T18:19:56,165 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 1ab6a75433d4cdfb37b6731cf70854ba 2024-11-18T18:19:56,179 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 1ab6a75433d4cdfb37b6731cf70854ba 2024-11-18T18:19:56,195 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnNextAsync/1ab6a75433d4cdfb37b6731cf70854ba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:19:56,197 INFO [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 1ab6a75433d4cdfb37b6731cf70854ba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64830570, jitterRate=-0.03394922614097595}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T18:19:56,197 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1ab6a75433d4cdfb37b6731cf70854ba 2024-11-18T18:19:56,198 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 1ab6a75433d4cdfb37b6731cf70854ba: Running coprocessor pre-open hook at 1731953996130Writing region info on filesystem at 1731953996130Initializing all the Stores at 1731953996138 (+8 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731953996138Cleaning up temporary data from old regions at 1731953996165 (+27 ms)Running coprocessor post-open hooks at 1731953996197 (+32 ms)Region opened successfully at 1731953996198 (+1 ms) 2024-11-18T18:19:56,201 INFO [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba., pid=12, masterSystemTime=1731953996112 2024-11-18T18:19:56,212 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba. 2024-11-18T18:19:56,212 INFO [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba. 2024-11-18T18:19:56,213 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=1ab6a75433d4cdfb37b6731cf70854ba, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:19:56,219 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ab6a75433d4cdfb37b6731cf70854ba, server=961a81da9827,38009,1731953921685 because future has completed 2024-11-18T18:19:56,242 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-18T18:19:56,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 1ab6a75433d4cdfb37b6731cf70854ba, server=961a81da9827,38009,1731953921685 in 277 msec 2024-11-18T18:19:56,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-18T18:19:56,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=1ab6a75433d4cdfb37b6731cf70854ba, ASSIGN in 455 msec 2024-11-18T18:19:56,252 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T18:19:56,253 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731953996252"}]},"ts":"1731953996252"} 2024-11-18T18:19:56,259 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-18T18:19:56,266 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T18:19:56,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 998 msec 2024-11-18T18:19:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-18T18:19:56,299 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-18T18:19:56,299 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-18T18:19:56,299 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:19:56,306 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-18T18:19:56,306 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:19:56,306 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-18T18:19:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba., hostname=961a81da9827,38009,1731953921685, seqNum=2] 2024-11-18T18:19:56,340 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-18T18:19:56,340 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-18T18:19:56,347 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-18T18:19:56,351 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-18T18:19:56,353 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-18T18:19:56,354 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38009 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-18T18:19:58,046 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-18T18:20:00,959 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-18T18:20:00,959 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-18T18:20:01,632 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T18:20:02,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38009 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '6208426143205294101' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-18T18:20:02,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38009 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:42496 deadline: 1731954008363, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '6208426143205294101' does not exist 2024-11-18T18:20:02,372 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 961a81da9827,38009,1731953921685 for closing scanner id = 6208426143205294101 for 1ab6a75433d4cdfb37b6731cf70854ba of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '6208426143205294101' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:20:02,376 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=220 (was 218) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=690 (was 690), ProcessCount=11 (was 11), AvailableMemoryMB=3072 (was 1832) - AvailableMemoryMB LEAK? - 2024-11-18T18:20:02,386 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=220, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=690, ProcessCount=11, AvailableMemoryMB=3068 2024-11-18T18:20:02,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T18:20:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-18T18:20:02,396 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T18:20:02,396 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:20:02,396 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-18T18:20:02,398 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T18:20:02,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T18:20:02,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741841_1017 (size=423) 2024-11-18T18:20:02,438 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0b55a66895220fe4244d54385a091298, NAME => 'testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23 2024-11-18T18:20:02,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741842_1018 (size=74) 2024-11-18T18:20:02,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-11-18T18:20:02,855 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:20:02,855 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 0b55a66895220fe4244d54385a091298, disabling compactions & flushes 2024-11-18T18:20:02,855 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298. 2024-11-18T18:20:02,855 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298. 2024-11-18T18:20:02,856 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298. after waiting 0 ms 2024-11-18T18:20:02,856 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298. 2024-11-18T18:20:02,856 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298. 2024-11-18T18:20:02,856 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0b55a66895220fe4244d54385a091298: Waiting for close lock at 1731954002855Disabling compacts and flushes for region at 1731954002855Disabling writes for close at 1731954002856 (+1 ms)Writing region close event to WAL at 1731954002856Closed at 1731954002856 2024-11-18T18:20:02,858 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T18:20:02,859 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731954002858"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954002858"}]},"ts":"1731954002858"} 2024-11-18T18:20:02,862 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T18:20:02,864 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T18:20:02,864 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731954002864"}]},"ts":"1731954002864"} 2024-11-18T18:20:02,868 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-18T18:20:02,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=0b55a66895220fe4244d54385a091298, ASSIGN}] 2024-11-18T18:20:02,877 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=0b55a66895220fe4244d54385a091298, ASSIGN 2024-11-18T18:20:02,881 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=0b55a66895220fe4244d54385a091298, ASSIGN; state=OFFLINE, location=961a81da9827,38009,1731953921685; forceNewPlan=false, retain=false 2024-11-18T18:20:03,034 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=0b55a66895220fe4244d54385a091298, regionState=OPENING, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:20:03,040 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=0b55a66895220fe4244d54385a091298, ASSIGN because future has completed 2024-11-18T18:20:03,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b55a66895220fe4244d54385a091298, server=961a81da9827,38009,1731953921685}] 2024-11-18T18:20:03,215 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298. 2024-11-18T18:20:03,215 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 0b55a66895220fe4244d54385a091298, NAME => 'testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298.', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:20:03,216 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 0b55a66895220fe4244d54385a091298 2024-11-18T18:20:03,216 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:20:03,216 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 0b55a66895220fe4244d54385a091298 2024-11-18T18:20:03,216 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 0b55a66895220fe4244d54385a091298 2024-11-18T18:20:03,223 INFO [StoreOpener-0b55a66895220fe4244d54385a091298-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0b55a66895220fe4244d54385a091298 2024-11-18T18:20:03,228 INFO [StoreOpener-0b55a66895220fe4244d54385a091298-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b55a66895220fe4244d54385a091298 columnFamilyName testFamily 2024-11-18T18:20:03,228 DEBUG [StoreOpener-0b55a66895220fe4244d54385a091298-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:20:03,230 INFO [StoreOpener-0b55a66895220fe4244d54385a091298-1 {}] regionserver.HStore(327): Store=0b55a66895220fe4244d54385a091298/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:20:03,230 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 0b55a66895220fe4244d54385a091298 2024-11-18T18:20:03,231 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testRetryOutOfOrderScannerNextException/0b55a66895220fe4244d54385a091298 2024-11-18T18:20:03,232 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testRetryOutOfOrderScannerNextException/0b55a66895220fe4244d54385a091298 2024-11-18T18:20:03,233 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 0b55a66895220fe4244d54385a091298 2024-11-18T18:20:03,233 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 0b55a66895220fe4244d54385a091298 2024-11-18T18:20:03,237 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 0b55a66895220fe4244d54385a091298 2024-11-18T18:20:03,241 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testRetryOutOfOrderScannerNextException/0b55a66895220fe4244d54385a091298/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:20:03,242 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 0b55a66895220fe4244d54385a091298; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70166074, jitterRate=0.04555597901344299}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T18:20:03,242 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0b55a66895220fe4244d54385a091298 2024-11-18T18:20:03,243 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 0b55a66895220fe4244d54385a091298: Running coprocessor pre-open hook at 1731954003216Writing region info on filesystem at 1731954003216Initializing all the Stores at 1731954003222 (+6 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954003222Cleaning up temporary data from old regions at 1731954003233 (+11 ms)Running coprocessor post-open hooks at 1731954003242 (+9 ms)Region opened successfully at 1731954003243 (+1 ms) 2024-11-18T18:20:03,247 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298., pid=15, masterSystemTime=1731954003206 2024-11-18T18:20:03,251 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298. 2024-11-18T18:20:03,251 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298. 2024-11-18T18:20:03,252 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=0b55a66895220fe4244d54385a091298, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:20:03,265 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b55a66895220fe4244d54385a091298, server=961a81da9827,38009,1731953921685 because future has completed 2024-11-18T18:20:03,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-18T18:20:03,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 0b55a66895220fe4244d54385a091298, server=961a81da9827,38009,1731953921685 in 228 msec 2024-11-18T18:20:03,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-18T18:20:03,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=0b55a66895220fe4244d54385a091298, ASSIGN in 405 msec 2024-11-18T18:20:03,280 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T18:20:03,280 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731954003280"}]},"ts":"1731954003280"} 2024-11-18T18:20:03,283 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-18T18:20:03,284 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T18:20:03,287 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 895 msec 2024-11-18T18:20:03,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T18:20:03,418 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-18T18:20:03,418 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-18T18:20:03,419 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:20:03,425 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-18T18:20:03,425 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:20:03,425 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-18T18:20:03,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298., hostname=961a81da9827,38009,1731953921685, seqNum=2] 2024-11-18T18:20:03,444 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-18T18:20:03,444 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-18T18:20:03,448 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-18T18:20:03,451 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-18T18:20:03,452 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-18T18:20:03,453 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=6208426143205294105 2024-11-18T18:20:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:42496 deadline: 1731954006452, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-18T18:20:03,456 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-18T18:20:03,457 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-18T18:20:03,458 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-18T18:20:03,459 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-18T18:20:03,460 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=6208426143205294107 2024-11-18T18:20:03,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:42496 deadline: 1731954006459, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-18T18:20:03,464 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-18T18:20:03,465 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=6208426143205294108 2024-11-18T18:20:03,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:42496 deadline: 1731954006464, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-18T18:20:03,472 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-18T18:20:03,473 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=6208426143205294109 2024-11-18T18:20:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:42496 deadline: 1731954006471, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-18T18:20:03,477 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-18T18:20:03,489 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 220) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-838902606_22 at /127.0.0.1:35490 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=690 (was 690), ProcessCount=11 (was 11), AvailableMemoryMB=2956 (was 3068) 2024-11-18T18:20:03,498 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=690, ProcessCount=11, AvailableMemoryMB=2955 2024-11-18T18:20:03,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T18:20:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-18T18:20:03,506 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T18:20:03,506 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:20:03,506 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-18T18:20:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-18T18:20:03,508 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T18:20:03,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741843_1019 (size=418) 2024-11-18T18:20:03,923 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e1407279fa27e618cab5cc5f365ded5a, NAME => 'testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23 2024-11-18T18:20:03,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741844_1020 (size=69) 2024-11-18T18:20:03,943 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:20:03,944 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing e1407279fa27e618cab5cc5f365ded5a, disabling compactions & flushes 2024-11-18T18:20:03,944 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a. 2024-11-18T18:20:03,944 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a. 2024-11-18T18:20:03,944 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a. after waiting 0 ms 2024-11-18T18:20:03,944 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a. 2024-11-18T18:20:03,944 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a. 2024-11-18T18:20:03,944 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for e1407279fa27e618cab5cc5f365ded5a: Waiting for close lock at 1731954003943Disabling compacts and flushes for region at 1731954003943Disabling writes for close at 1731954003944 (+1 ms)Writing region close event to WAL at 1731954003944Closed at 1731954003944 2024-11-18T18:20:03,946 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T18:20:03,947 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1731954003946"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954003946"}]},"ts":"1731954003946"} 2024-11-18T18:20:03,952 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T18:20:03,954 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T18:20:03,954 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731954003954"}]},"ts":"1731954003954"} 2024-11-18T18:20:03,961 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-18T18:20:03,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=e1407279fa27e618cab5cc5f365ded5a, ASSIGN}] 2024-11-18T18:20:03,964 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=e1407279fa27e618cab5cc5f365ded5a, ASSIGN 2024-11-18T18:20:03,965 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=e1407279fa27e618cab5cc5f365ded5a, ASSIGN; state=OFFLINE, location=961a81da9827,38009,1731953921685; forceNewPlan=false, retain=false 2024-11-18T18:20:04,047 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-18T18:20:04,116 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=e1407279fa27e618cab5cc5f365ded5a, regionState=OPENING, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:20:04,120 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=e1407279fa27e618cab5cc5f365ded5a, ASSIGN because future has completed 2024-11-18T18:20:04,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1407279fa27e618cab5cc5f365ded5a, server=961a81da9827,38009,1731953921685}] 2024-11-18T18:20:04,281 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a. 2024-11-18T18:20:04,282 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => e1407279fa27e618cab5cc5f365ded5a, NAME => 'testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a.', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:20:04,282 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner e1407279fa27e618cab5cc5f365ded5a 2024-11-18T18:20:04,282 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:20:04,282 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for e1407279fa27e618cab5cc5f365ded5a 2024-11-18T18:20:04,282 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for e1407279fa27e618cab5cc5f365ded5a 2024-11-18T18:20:04,286 INFO [StoreOpener-e1407279fa27e618cab5cc5f365ded5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e1407279fa27e618cab5cc5f365ded5a 2024-11-18T18:20:04,289 INFO [StoreOpener-e1407279fa27e618cab5cc5f365ded5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1407279fa27e618cab5cc5f365ded5a columnFamilyName testFamily 2024-11-18T18:20:04,290 DEBUG [StoreOpener-e1407279fa27e618cab5cc5f365ded5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:20:04,291 INFO [StoreOpener-e1407279fa27e618cab5cc5f365ded5a-1 {}] regionserver.HStore(327): Store=e1407279fa27e618cab5cc5f365ded5a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:20:04,291 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for e1407279fa27e618cab5cc5f365ded5a 2024-11-18T18:20:04,293 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnOpenScanner/e1407279fa27e618cab5cc5f365ded5a 2024-11-18T18:20:04,294 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnOpenScanner/e1407279fa27e618cab5cc5f365ded5a 2024-11-18T18:20:04,294 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for e1407279fa27e618cab5cc5f365ded5a 2024-11-18T18:20:04,294 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for e1407279fa27e618cab5cc5f365ded5a 2024-11-18T18:20:04,299 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for e1407279fa27e618cab5cc5f365ded5a 2024-11-18T18:20:04,303 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnOpenScanner/e1407279fa27e618cab5cc5f365ded5a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:20:04,304 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened e1407279fa27e618cab5cc5f365ded5a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60458637, jitterRate=-0.09909610450267792}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T18:20:04,304 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e1407279fa27e618cab5cc5f365ded5a 2024-11-18T18:20:04,305 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for e1407279fa27e618cab5cc5f365ded5a: Running coprocessor pre-open hook at 1731954004283Writing region info on filesystem at 1731954004283Initializing all the Stores at 1731954004284 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954004284Cleaning up temporary data from old regions at 1731954004294 (+10 ms)Running coprocessor post-open hooks at 1731954004304 (+10 ms)Region opened successfully at 1731954004305 (+1 ms) 2024-11-18T18:20:04,309 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a., pid=18, masterSystemTime=1731954004274 2024-11-18T18:20:04,313 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a. 2024-11-18T18:20:04,313 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a. 2024-11-18T18:20:04,314 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=e1407279fa27e618cab5cc5f365ded5a, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:20:04,318 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1407279fa27e618cab5cc5f365ded5a, server=961a81da9827,38009,1731953921685 because future has completed 2024-11-18T18:20:04,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-18T18:20:04,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure e1407279fa27e618cab5cc5f365ded5a, server=961a81da9827,38009,1731953921685 in 199 msec 2024-11-18T18:20:04,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-18T18:20:04,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=e1407279fa27e618cab5cc5f365ded5a, ASSIGN in 362 msec 2024-11-18T18:20:04,329 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T18:20:04,329 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731954004329"}]},"ts":"1731954004329"} 2024-11-18T18:20:04,332 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-18T18:20:04,333 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T18:20:04,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 831 msec 2024-11-18T18:20:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-18T18:20:04,528 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-18T18:20:04,528 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-18T18:20:04,528 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:20:04,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-18T18:20:04,534 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:20:04,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-18T18:20:04,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a., hostname=961a81da9827,38009,1731953921685, seqNum=2] 2024-11-18T18:20:04,555 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-18T18:20:04,555 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-18T18:20:04,560 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-18T18:20:04,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-18T18:20:05,562 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a., hostname=961a81da9827,38009,1731953921685, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a., hostname=961a81da9827,38009,1731953921685, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=961a81da9827:38009 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-18T18:20:05,562 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a., hostname=961a81da9827,38009,1731953921685, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-18T18:20:05,562 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a., hostname=961a81da9827,38009,1731953921685, seqNum=2 from cache 2024-11-18T18:20:06,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-11-18T18:20:06,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a., hostname=961a81da9827,38009,1731953921685, seqNum=2] 2024-11-18T18:20:06,581 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-18T18:20:07,562 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a., hostname=961a81da9827,38009,1731953921685, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a., hostname=961a81da9827,38009,1731953921685, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=961a81da9827:38009 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=982ms, rpcTimeout=980ms 2024-11-18T18:20:07,562 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a., hostname=961a81da9827,38009,1731953921685, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=982ms, rpcTimeout=980ms 2024-11-18T18:20:07,562 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a., hostname=961a81da9827,38009,1731953921685, seqNum=2 from cache 2024-11-18T18:20:07,577 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223 (was 223), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=659 (was 690), ProcessCount=11 (was 11), AvailableMemoryMB=2714 (was 2955) 2024-11-18T18:20:07,590 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=659, ProcessCount=11, AvailableMemoryMB=2715 2024-11-18T18:20:07,593 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T18:20:07,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-18T18:20:07,597 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T18:20:07,597 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:20:07,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-18T18:20:07,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T18:20:07,599 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T18:20:07,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741845_1021 (size=416) 2024-11-18T18:20:08,021 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ed0869c8914ca2d0e3dc8322a35c1663, NAME => 'testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23 2024-11-18T18:20:08,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741846_1022 (size=67) 2024-11-18T18:20:08,059 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:20:08,059 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing ed0869c8914ca2d0e3dc8322a35c1663, disabling compactions & flushes 2024-11-18T18:20:08,059 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663. 2024-11-18T18:20:08,059 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663. 2024-11-18T18:20:08,060 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663. after waiting 0 ms 2024-11-18T18:20:08,060 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663. 2024-11-18T18:20:08,060 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663. 2024-11-18T18:20:08,060 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for ed0869c8914ca2d0e3dc8322a35c1663: Waiting for close lock at 1731954008059Disabling compacts and flushes for region at 1731954008059Disabling writes for close at 1731954008060 (+1 ms)Writing region close event to WAL at 1731954008060Closed at 1731954008060 2024-11-18T18:20:08,063 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T18:20:08,063 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731954008063"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954008063"}]},"ts":"1731954008063"} 2024-11-18T18:20:08,070 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T18:20:08,074 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T18:20:08,074 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731954008074"}]},"ts":"1731954008074"} 2024-11-18T18:20:08,078 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-18T18:20:08,078 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=ed0869c8914ca2d0e3dc8322a35c1663, ASSIGN}] 2024-11-18T18:20:08,080 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=ed0869c8914ca2d0e3dc8322a35c1663, ASSIGN 2024-11-18T18:20:08,081 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=ed0869c8914ca2d0e3dc8322a35c1663, ASSIGN; state=OFFLINE, location=961a81da9827,38009,1731953921685; forceNewPlan=false, retain=false 2024-11-18T18:20:08,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-11-18T18:20:08,233 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=ed0869c8914ca2d0e3dc8322a35c1663, regionState=OPENING, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:20:08,244 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=ed0869c8914ca2d0e3dc8322a35c1663, ASSIGN because future has completed 2024-11-18T18:20:08,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed0869c8914ca2d0e3dc8322a35c1663, server=961a81da9827,38009,1731953921685}] 2024-11-18T18:20:08,424 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663. 2024-11-18T18:20:08,425 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => ed0869c8914ca2d0e3dc8322a35c1663, NAME => 'testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663.', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:20:08,425 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:08,426 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:20:08,426 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:08,426 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:08,437 INFO [StoreOpener-ed0869c8914ca2d0e3dc8322a35c1663-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:08,441 INFO [StoreOpener-ed0869c8914ca2d0e3dc8322a35c1663-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed0869c8914ca2d0e3dc8322a35c1663 columnFamilyName testFamily 2024-11-18T18:20:08,441 DEBUG [StoreOpener-ed0869c8914ca2d0e3dc8322a35c1663-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:20:08,443 INFO [StoreOpener-ed0869c8914ca2d0e3dc8322a35c1663-1 {}] regionserver.HStore(327): Store=ed0869c8914ca2d0e3dc8322a35c1663/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:20:08,444 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:08,445 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnNextAsync/ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:08,446 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnNextAsync/ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:08,458 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:08,458 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:08,486 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:08,510 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnNextAsync/ed0869c8914ca2d0e3dc8322a35c1663/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:20:08,514 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened ed0869c8914ca2d0e3dc8322a35c1663; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64068051, jitterRate=-0.04531164467334747}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T18:20:08,514 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:08,515 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for ed0869c8914ca2d0e3dc8322a35c1663: Running coprocessor pre-open hook at 1731954008426Writing region info on filesystem at 1731954008426Initializing all the Stores at 1731954008431 (+5 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954008431Cleaning up temporary data from old regions at 1731954008458 (+27 ms)Running coprocessor post-open hooks at 1731954008514 (+56 ms)Region opened successfully at 1731954008515 (+1 ms) 2024-11-18T18:20:08,517 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663., pid=21, masterSystemTime=1731954008404 2024-11-18T18:20:08,524 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663. 2024-11-18T18:20:08,525 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663. 2024-11-18T18:20:08,526 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=ed0869c8914ca2d0e3dc8322a35c1663, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:20:08,532 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed0869c8914ca2d0e3dc8322a35c1663, server=961a81da9827,38009,1731953921685 because future has completed 2024-11-18T18:20:08,540 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-18T18:20:08,540 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure ed0869c8914ca2d0e3dc8322a35c1663, server=961a81da9827,38009,1731953921685 in 289 msec 2024-11-18T18:20:08,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-18T18:20:08,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=ed0869c8914ca2d0e3dc8322a35c1663, ASSIGN in 463 msec 2024-11-18T18:20:08,547 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T18:20:08,547 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731954008547"}]},"ts":"1731954008547"} 2024-11-18T18:20:08,551 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-18T18:20:08,552 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T18:20:08,555 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 959 msec 2024-11-18T18:20:08,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T18:20:08,618 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-18T18:20:08,618 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-18T18:20:08,618 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:20:08,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-18T18:20:08,624 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:20:08,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-18T18:20:08,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663., hostname=961a81da9827,38009,1731953921685, seqNum=2] 2024-11-18T18:20:08,656 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-18T18:20:08,657 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-18T18:20:08,661 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-18T18:20:08,664 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-18T18:20:08,666 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-18T18:20:08,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-18T18:20:08,719 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T18:20:09,491 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T18:20:10,049 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-18T18:20:10,050 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-18T18:20:10,959 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-18T18:20:10,959 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-18T18:20:10,960 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-18T18:20:10,960 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-18T18:20:10,961 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-18T18:20:10,961 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-18T18:20:11,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38009 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '6208426143205294120' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-18T18:20:11,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38009 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:42496 deadline: 1731954012673, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '6208426143205294120' does not exist 2024-11-18T18:20:11,676 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 961a81da9827,38009,1731953921685 for closing scanner id = 6208426143205294120 for ed0869c8914ca2d0e3dc8322a35c1663 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '6208426143205294120' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:20:11,684 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=223 (was 223), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=686 (was 659) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2564 (was 2715) 2024-11-18T18:20:11,692 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=686, ProcessCount=11, AvailableMemoryMB=2564 2024-11-18T18:20:11,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T18:20:11,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-18T18:20:11,699 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T18:20:11,699 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:20:11,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-18T18:20:11,700 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T18:20:11,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-18T18:20:11,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741847_1023 (size=411) 2024-11-18T18:20:12,117 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 45e5d79edf6587776c61c37017185e82, NAME => 'testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23 2024-11-18T18:20:12,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741848_1024 (size=62) 2024-11-18T18:20:12,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-11-18T18:20:12,527 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:20:12,527 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 45e5d79edf6587776c61c37017185e82, disabling compactions & flushes 2024-11-18T18:20:12,527 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82. 2024-11-18T18:20:12,527 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82. 2024-11-18T18:20:12,527 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82. after waiting 0 ms 2024-11-18T18:20:12,527 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82. 2024-11-18T18:20:12,527 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82. 2024-11-18T18:20:12,528 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 45e5d79edf6587776c61c37017185e82: Waiting for close lock at 1731954012527Disabling compacts and flushes for region at 1731954012527Disabling writes for close at 1731954012527Writing region close event to WAL at 1731954012527Closed at 1731954012527 2024-11-18T18:20:12,530 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T18:20:12,530 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731954012530"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954012530"}]},"ts":"1731954012530"} 2024-11-18T18:20:12,534 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T18:20:12,536 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T18:20:12,536 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731954012536"}]},"ts":"1731954012536"} 2024-11-18T18:20:12,539 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-18T18:20:12,539 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=45e5d79edf6587776c61c37017185e82, ASSIGN}] 2024-11-18T18:20:12,541 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=45e5d79edf6587776c61c37017185e82, ASSIGN 2024-11-18T18:20:12,543 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=45e5d79edf6587776c61c37017185e82, ASSIGN; state=OFFLINE, location=961a81da9827,38009,1731953921685; forceNewPlan=false, retain=false 2024-11-18T18:20:12,694 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=45e5d79edf6587776c61c37017185e82, regionState=OPENING, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:20:12,698 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=45e5d79edf6587776c61c37017185e82, ASSIGN because future has completed 2024-11-18T18:20:12,698 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e5d79edf6587776c61c37017185e82, server=961a81da9827,38009,1731953921685}] 2024-11-18T18:20:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-18T18:20:12,859 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82. 2024-11-18T18:20:12,859 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 45e5d79edf6587776c61c37017185e82, NAME => 'testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82.', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:20:12,860 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 45e5d79edf6587776c61c37017185e82 2024-11-18T18:20:12,860 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:20:12,860 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 45e5d79edf6587776c61c37017185e82 2024-11-18T18:20:12,860 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 45e5d79edf6587776c61c37017185e82 2024-11-18T18:20:12,863 INFO [StoreOpener-45e5d79edf6587776c61c37017185e82-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 45e5d79edf6587776c61c37017185e82 2024-11-18T18:20:12,866 INFO [StoreOpener-45e5d79edf6587776c61c37017185e82-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45e5d79edf6587776c61c37017185e82 columnFamilyName testFamily 2024-11-18T18:20:12,866 DEBUG [StoreOpener-45e5d79edf6587776c61c37017185e82-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:20:12,867 INFO [StoreOpener-45e5d79edf6587776c61c37017185e82-1 {}] regionserver.HStore(327): Store=45e5d79edf6587776c61c37017185e82/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:20:12,867 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 45e5d79edf6587776c61c37017185e82 2024-11-18T18:20:12,868 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnNext/45e5d79edf6587776c61c37017185e82 2024-11-18T18:20:12,869 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnNext/45e5d79edf6587776c61c37017185e82 2024-11-18T18:20:12,870 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 45e5d79edf6587776c61c37017185e82 2024-11-18T18:20:12,870 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 45e5d79edf6587776c61c37017185e82 2024-11-18T18:20:12,873 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 45e5d79edf6587776c61c37017185e82 2024-11-18T18:20:12,882 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnNext/45e5d79edf6587776c61c37017185e82/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:20:12,883 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 45e5d79edf6587776c61c37017185e82; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69662145, jitterRate=0.03804685175418854}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T18:20:12,884 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 45e5d79edf6587776c61c37017185e82 2024-11-18T18:20:12,884 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 45e5d79edf6587776c61c37017185e82: Running coprocessor pre-open hook at 1731954012861Writing region info on filesystem at 1731954012861Initializing all the Stores at 1731954012862 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954012862Cleaning up temporary data from old regions at 1731954012870 (+8 ms)Running coprocessor post-open hooks at 1731954012884 (+14 ms)Region opened successfully at 1731954012884 2024-11-18T18:20:12,886 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82., pid=24, masterSystemTime=1731954012852 2024-11-18T18:20:12,891 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82. 2024-11-18T18:20:12,891 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82. 2024-11-18T18:20:12,892 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=45e5d79edf6587776c61c37017185e82, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:20:12,900 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e5d79edf6587776c61c37017185e82, server=961a81da9827,38009,1731953921685 because future has completed 2024-11-18T18:20:12,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-18T18:20:12,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 45e5d79edf6587776c61c37017185e82, server=961a81da9827,38009,1731953921685 in 204 msec 2024-11-18T18:20:12,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-18T18:20:12,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=45e5d79edf6587776c61c37017185e82, ASSIGN in 366 msec 2024-11-18T18:20:12,910 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T18:20:12,910 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731954012910"}]},"ts":"1731954012910"} 2024-11-18T18:20:12,913 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-18T18:20:12,914 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T18:20:12,916 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2190 sec 2024-11-18T18:20:13,928 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T18:20:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-18T18:20:14,728 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-18T18:20:14,728 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-18T18:20:14,729 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:20:14,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-18T18:20:14,739 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:20:14,740 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-18T18:20:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82., hostname=961a81da9827,38009,1731953921685, seqNum=2] 2024-11-18T18:20:14,775 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-18T18:20:14,775 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-18T18:20:14,780 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-18T18:20:14,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-18T18:20:14,788 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-18T18:20:14,788 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-18T18:20:16,052 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-18T18:20:17,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38009 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '6208426143205294124' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-18T18:20:17,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38009 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:42496 deadline: 1731954018793, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '6208426143205294124' does not exist 2024-11-18T18:20:17,795 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 961a81da9827,38009,1731953921685 for closing scanner id = 6208426143205294124 for 45e5d79edf6587776c61c37017185e82 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '6208426143205294124' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:20:17,818 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 223), OpenFileDescriptor=448 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=711 (was 686) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2415 (was 2564) 2024-11-18T18:20:17,835 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=711, ProcessCount=11, AvailableMemoryMB=2411 2024-11-18T18:20:17,839 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39417 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T18:20:17,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39417 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-18T18:20:17,858 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T18:20:17,865 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:20:17,867 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39417 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-18T18:20:17,868 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T18:20:17,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T18:20:17,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741849_1025 (size=419) 2024-11-18T18:20:18,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-11-18T18:20:18,348 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c464a047be111daf460feab0aed90107, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23 2024-11-18T18:20:18,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741850_1026 (size=70) 2024-11-18T18:20:18,391 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:20:18,391 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing c464a047be111daf460feab0aed90107, disabling compactions & flushes 2024-11-18T18:20:18,391 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107. 2024-11-18T18:20:18,391 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107. 2024-11-18T18:20:18,391 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107. after waiting 0 ms 2024-11-18T18:20:18,391 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107. 2024-11-18T18:20:18,392 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107. 2024-11-18T18:20:18,392 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for c464a047be111daf460feab0aed90107: Waiting for close lock at 1731954018391Disabling compacts and flushes for region at 1731954018391Disabling writes for close at 1731954018391Writing region close event to WAL at 1731954018391Closed at 1731954018391 2024-11-18T18:20:18,404 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T18:20:18,404 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1731954018404"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954018404"}]},"ts":"1731954018404"} 2024-11-18T18:20:18,433 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T18:20:18,445 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T18:20:18,446 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731954018446"}]},"ts":"1731954018446"} 2024-11-18T18:20:18,465 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-18T18:20:18,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=c464a047be111daf460feab0aed90107, ASSIGN}] 2024-11-18T18:20:18,478 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=c464a047be111daf460feab0aed90107, ASSIGN 2024-11-18T18:20:18,485 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=c464a047be111daf460feab0aed90107, ASSIGN; state=OFFLINE, location=961a81da9827,38009,1731953921685; forceNewPlan=false, retain=false 2024-11-18T18:20:18,642 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=c464a047be111daf460feab0aed90107, regionState=OPENING, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:20:18,646 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=c464a047be111daf460feab0aed90107, ASSIGN because future has completed 2024-11-18T18:20:18,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure c464a047be111daf460feab0aed90107, server=961a81da9827,38009,1731953921685}] 2024-11-18T18:20:18,843 INFO [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107. 2024-11-18T18:20:18,844 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => c464a047be111daf460feab0aed90107, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107.', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:20:18,844 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync c464a047be111daf460feab0aed90107 2024-11-18T18:20:18,844 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:20:18,845 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for c464a047be111daf460feab0aed90107 2024-11-18T18:20:18,845 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for c464a047be111daf460feab0aed90107 2024-11-18T18:20:18,859 INFO [StoreOpener-c464a047be111daf460feab0aed90107-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c464a047be111daf460feab0aed90107 2024-11-18T18:20:18,878 INFO [StoreOpener-c464a047be111daf460feab0aed90107-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c464a047be111daf460feab0aed90107 columnFamilyName testFamily 2024-11-18T18:20:18,878 DEBUG [StoreOpener-c464a047be111daf460feab0aed90107-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:20:18,879 INFO [StoreOpener-c464a047be111daf460feab0aed90107-1 {}] regionserver.HStore(327): Store=c464a047be111daf460feab0aed90107/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:20:18,880 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for c464a047be111daf460feab0aed90107 2024-11-18T18:20:18,881 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/c464a047be111daf460feab0aed90107 2024-11-18T18:20:18,882 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/c464a047be111daf460feab0aed90107 2024-11-18T18:20:18,883 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for c464a047be111daf460feab0aed90107 2024-11-18T18:20:18,883 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for c464a047be111daf460feab0aed90107 2024-11-18T18:20:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T18:20:18,889 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for c464a047be111daf460feab0aed90107 2024-11-18T18:20:18,907 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/c464a047be111daf460feab0aed90107/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:20:18,910 INFO [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened c464a047be111daf460feab0aed90107; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60773735, jitterRate=-0.09440077841281891}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T18:20:18,910 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c464a047be111daf460feab0aed90107 2024-11-18T18:20:18,910 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for c464a047be111daf460feab0aed90107: Running coprocessor pre-open hook at 1731954018845Writing region info on filesystem at 1731954018845Initializing all the Stores at 1731954018857 (+12 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954018857Cleaning up temporary data from old regions at 1731954018883 (+26 ms)Running coprocessor post-open hooks at 1731954018910 (+27 ms)Region opened successfully at 1731954018910 2024-11-18T18:20:18,912 INFO [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107., pid=27, masterSystemTime=1731954018808 2024-11-18T18:20:18,915 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107. 2024-11-18T18:20:18,915 INFO [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107. 2024-11-18T18:20:18,916 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=c464a047be111daf460feab0aed90107, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:20:18,920 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure c464a047be111daf460feab0aed90107, server=961a81da9827,38009,1731953921685 because future has completed 2024-11-18T18:20:18,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-18T18:20:18,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure c464a047be111daf460feab0aed90107, server=961a81da9827,38009,1731953921685 in 275 msec 2024-11-18T18:20:18,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-18T18:20:18,943 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=c464a047be111daf460feab0aed90107, ASSIGN in 459 msec 2024-11-18T18:20:18,944 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T18:20:18,944 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731954018944"}]},"ts":"1731954018944"} 2024-11-18T18:20:18,948 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-18T18:20:18,949 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T18:20:18,952 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.1100 sec 2024-11-18T18:20:20,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-18T18:20:20,899 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-18T18:20:20,899 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-18T18:20:20,899 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:20:20,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-18T18:20:20,910 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:20:20,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-18T18:20:20,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107., hostname=961a81da9827,38009,1731953921685, seqNum=2] 2024-11-18T18:20:20,932 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-18T18:20:20,932 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-18T18:20:20,938 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-18T18:20:20,940 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38009 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-18T18:20:20,959 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-18T18:20:20,959 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-18T18:20:20,960 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-18T18:20:20,960 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-18T18:20:22,053 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-18T18:20:24,347 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T18:20:26,784 INFO [master/961a81da9827:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-18T18:20:26,784 INFO [master/961a81da9827:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-18T18:20:26,946 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107., hostname=961a81da9827,38009,1731953921685, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107., hostname=961a81da9827,38009,1731953921685, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=961a81da9827:38009 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6007ms, rpcTimeout=5999ms 2024-11-18T18:20:26,946 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107., hostname=961a81da9827,38009,1731953921685, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6007ms, rpcTimeout=5999ms 2024-11-18T18:20:26,946 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107., hostname=961a81da9827,38009,1731953921685, seqNum=2 from cache 2024-11-18T18:20:26,985 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=746 (was 711) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1937 (was 2411) 2024-11-18T18:20:27,007 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=746, ProcessCount=11, AvailableMemoryMB=1936 2024-11-18T18:20:27,011 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39417 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T18:20:27,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39417 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-18T18:20:27,022 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T18:20:27,022 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:20:27,023 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39417 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-18T18:20:27,024 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T18:20:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-18T18:20:27,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741851_1027 (size=407) 2024-11-18T18:20:27,110 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 169a83608063e417ccb48f2d9f0ed8d3, NAME => 'hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23 2024-11-18T18:20:27,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741852_1028 (size=58) 2024-11-18T18:20:27,157 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:20:27,158 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 169a83608063e417ccb48f2d9f0ed8d3, disabling compactions & flushes 2024-11-18T18:20:27,158 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3. 2024-11-18T18:20:27,158 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3. 2024-11-18T18:20:27,158 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3. after waiting 0 ms 2024-11-18T18:20:27,158 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3. 2024-11-18T18:20:27,158 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3. 2024-11-18T18:20:27,158 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 169a83608063e417ccb48f2d9f0ed8d3: Waiting for close lock at 1731954027158Disabling compacts and flushes for region at 1731954027158Disabling writes for close at 1731954027158Writing region close event to WAL at 1731954027158Closed at 1731954027158 2024-11-18T18:20:27,163 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T18:20:27,164 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1731954027163"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954027163"}]},"ts":"1731954027163"} 2024-11-18T18:20:27,172 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T18:20:27,174 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T18:20:27,174 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731954027174"}]},"ts":"1731954027174"} 2024-11-18T18:20:27,179 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-18T18:20:27,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=169a83608063e417ccb48f2d9f0ed8d3, ASSIGN}] 2024-11-18T18:20:27,182 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=169a83608063e417ccb48f2d9f0ed8d3, ASSIGN 2024-11-18T18:20:27,185 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=169a83608063e417ccb48f2d9f0ed8d3, ASSIGN; state=OFFLINE, location=961a81da9827,38009,1731953921685; forceNewPlan=false, retain=false 2024-11-18T18:20:27,336 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=169a83608063e417ccb48f2d9f0ed8d3, regionState=OPENING, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:20:27,342 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=169a83608063e417ccb48f2d9f0ed8d3, ASSIGN because future has completed 2024-11-18T18:20:27,374 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 169a83608063e417ccb48f2d9f0ed8d3, server=961a81da9827,38009,1731953921685}] 2024-11-18T18:20:27,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-11-18T18:20:27,558 INFO [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3. 2024-11-18T18:20:27,562 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 169a83608063e417ccb48f2d9f0ed8d3, NAME => 'hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3.', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:20:27,562 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 169a83608063e417ccb48f2d9f0ed8d3 2024-11-18T18:20:27,562 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:20:27,563 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 169a83608063e417ccb48f2d9f0ed8d3 2024-11-18T18:20:27,563 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 169a83608063e417ccb48f2d9f0ed8d3 2024-11-18T18:20:27,567 INFO [StoreOpener-169a83608063e417ccb48f2d9f0ed8d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 169a83608063e417ccb48f2d9f0ed8d3 2024-11-18T18:20:27,574 INFO [StoreOpener-169a83608063e417ccb48f2d9f0ed8d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 169a83608063e417ccb48f2d9f0ed8d3 columnFamilyName testFamily 2024-11-18T18:20:27,574 DEBUG [StoreOpener-169a83608063e417ccb48f2d9f0ed8d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:20:27,575 INFO [StoreOpener-169a83608063e417ccb48f2d9f0ed8d3-1 {}] regionserver.HStore(327): Store=169a83608063e417ccb48f2d9f0ed8d3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:20:27,576 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 169a83608063e417ccb48f2d9f0ed8d3 2024-11-18T18:20:27,577 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnNext/169a83608063e417ccb48f2d9f0ed8d3 2024-11-18T18:20:27,578 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnNext/169a83608063e417ccb48f2d9f0ed8d3 2024-11-18T18:20:27,579 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 169a83608063e417ccb48f2d9f0ed8d3 2024-11-18T18:20:27,580 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 169a83608063e417ccb48f2d9f0ed8d3 2024-11-18T18:20:27,584 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 169a83608063e417ccb48f2d9f0ed8d3 2024-11-18T18:20:27,588 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnNext/169a83608063e417ccb48f2d9f0ed8d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:20:27,589 INFO [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 169a83608063e417ccb48f2d9f0ed8d3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61575247, jitterRate=-0.08245731890201569}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T18:20:27,590 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 169a83608063e417ccb48f2d9f0ed8d3 2024-11-18T18:20:27,591 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 169a83608063e417ccb48f2d9f0ed8d3: Running coprocessor pre-open hook at 1731954027563Writing region info on filesystem at 1731954027563Initializing all the Stores at 1731954027567 (+4 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954027567Cleaning up temporary data from old regions at 1731954027580 (+13 ms)Running coprocessor post-open hooks at 1731954027590 (+10 ms)Region opened successfully at 1731954027591 (+1 ms) 2024-11-18T18:20:27,598 INFO [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3., pid=30, masterSystemTime=1731954027532 2024-11-18T18:20:27,603 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3. 2024-11-18T18:20:27,603 INFO [RS_OPEN_PRIORITY_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3. 2024-11-18T18:20:27,603 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=169a83608063e417ccb48f2d9f0ed8d3, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:20:27,609 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 169a83608063e417ccb48f2d9f0ed8d3, server=961a81da9827,38009,1731953921685 because future has completed 2024-11-18T18:20:27,614 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-18T18:20:27,614 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 169a83608063e417ccb48f2d9f0ed8d3, server=961a81da9827,38009,1731953921685 in 238 msec 2024-11-18T18:20:27,621 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-18T18:20:27,621 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=169a83608063e417ccb48f2d9f0ed8d3, ASSIGN in 436 msec 2024-11-18T18:20:27,622 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T18:20:27,623 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731954027622"}]},"ts":"1731954027622"} 2024-11-18T18:20:27,626 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-18T18:20:27,627 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T18:20:27,633 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 616 msec 2024-11-18T18:20:28,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-18T18:20:28,038 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-18T18:20:28,041 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-18T18:20:28,042 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:20:28,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-18T18:20:28,055 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-18T18:20:28,061 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:20:28,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-18T18:20:28,066 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3., hostname=961a81da9827,38009,1731953921685, seqNum=2] 2024-11-18T18:20:28,091 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-18T18:20:28,091 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-18T18:20:28,096 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-18T18:20:28,105 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38009 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-18T18:20:28,105 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-18T18:20:28,105 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-18T18:20:30,959 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-18T18:20:30,959 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-18T18:20:33,065 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T18:20:34,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38009 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '6208426143205294132' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-18T18:20:34,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38009 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:42496 deadline: 1731954040113, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '6208426143205294132' does not exist 2024-11-18T18:20:34,120 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 961a81da9827,38009,1731953921685 for closing scanner id = 6208426143205294132 for 169a83608063e417ccb48f2d9f0ed8d3 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '6208426143205294132' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:20:34,128 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=784 (was 746) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1465 (was 1936) 2024-11-18T18:20:34,139 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=784, ProcessCount=11, AvailableMemoryMB=1465 2024-11-18T18:20:34,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T18:20:34,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-18T18:20:34,146 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T18:20:34,147 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:20:34,147 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-18T18:20:34,148 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T18:20:34,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-18T18:20:34,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741853_1029 (size=423) 2024-11-18T18:20:34,178 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 12222987219b43598fd47740805cf6b1, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23 2024-11-18T18:20:34,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741854_1030 (size=74) 2024-11-18T18:20:34,199 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:20:34,199 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 12222987219b43598fd47740805cf6b1, disabling compactions & flushes 2024-11-18T18:20:34,199 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1. 2024-11-18T18:20:34,199 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1. 2024-11-18T18:20:34,199 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1. after waiting 0 ms 2024-11-18T18:20:34,199 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1. 2024-11-18T18:20:34,199 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1. 2024-11-18T18:20:34,199 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 12222987219b43598fd47740805cf6b1: Waiting for close lock at 1731954034199Disabling compacts and flushes for region at 1731954034199Disabling writes for close at 1731954034199Writing region close event to WAL at 1731954034199Closed at 1731954034199 2024-11-18T18:20:34,201 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T18:20:34,202 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731954034201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731954034201"}]},"ts":"1731954034201"} 2024-11-18T18:20:34,205 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T18:20:34,207 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T18:20:34,207 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731954034207"}]},"ts":"1731954034207"} 2024-11-18T18:20:34,211 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-18T18:20:34,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=12222987219b43598fd47740805cf6b1, ASSIGN}] 2024-11-18T18:20:34,213 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=12222987219b43598fd47740805cf6b1, ASSIGN 2024-11-18T18:20:34,215 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=12222987219b43598fd47740805cf6b1, ASSIGN; state=OFFLINE, location=961a81da9827,38009,1731953921685; forceNewPlan=false, retain=false 2024-11-18T18:20:34,366 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=12222987219b43598fd47740805cf6b1, regionState=OPENING, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:20:34,372 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=12222987219b43598fd47740805cf6b1, ASSIGN because future has completed 2024-11-18T18:20:34,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12222987219b43598fd47740805cf6b1, server=961a81da9827,38009,1731953921685}] 2024-11-18T18:20:34,549 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1. 2024-11-18T18:20:34,549 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 12222987219b43598fd47740805cf6b1, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1.', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:20:34,549 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 12222987219b43598fd47740805cf6b1 2024-11-18T18:20:34,550 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:20:34,550 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 12222987219b43598fd47740805cf6b1 2024-11-18T18:20:34,550 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 12222987219b43598fd47740805cf6b1 2024-11-18T18:20:34,566 INFO [StoreOpener-12222987219b43598fd47740805cf6b1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 12222987219b43598fd47740805cf6b1 2024-11-18T18:20:34,569 INFO [StoreOpener-12222987219b43598fd47740805cf6b1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12222987219b43598fd47740805cf6b1 columnFamilyName testFamily 2024-11-18T18:20:34,570 DEBUG [StoreOpener-12222987219b43598fd47740805cf6b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:20:34,582 INFO [StoreOpener-12222987219b43598fd47740805cf6b1-1 {}] regionserver.HStore(327): Store=12222987219b43598fd47740805cf6b1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:20:34,582 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 12222987219b43598fd47740805cf6b1 2024-11-18T18:20:34,584 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnOpenScannerAsync/12222987219b43598fd47740805cf6b1 2024-11-18T18:20:34,585 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnOpenScannerAsync/12222987219b43598fd47740805cf6b1 2024-11-18T18:20:34,586 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 12222987219b43598fd47740805cf6b1 2024-11-18T18:20:34,586 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 12222987219b43598fd47740805cf6b1 2024-11-18T18:20:34,590 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 12222987219b43598fd47740805cf6b1 2024-11-18T18:20:34,597 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnOpenScannerAsync/12222987219b43598fd47740805cf6b1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:20:34,598 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 12222987219b43598fd47740805cf6b1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67138632, jitterRate=4.4357776641845703E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T18:20:34,598 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 12222987219b43598fd47740805cf6b1 2024-11-18T18:20:34,599 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 12222987219b43598fd47740805cf6b1: Running coprocessor pre-open hook at 1731954034550Writing region info on filesystem at 1731954034550Initializing all the Stores at 1731954034554 (+4 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731954034554Cleaning up temporary data from old regions at 1731954034586 (+32 ms)Running coprocessor post-open hooks at 1731954034598 (+12 ms)Region opened successfully at 1731954034599 (+1 ms) 2024-11-18T18:20:34,601 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1., pid=33, masterSystemTime=1731954034531 2024-11-18T18:20:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-11-18T18:20:34,611 DEBUG [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1. 2024-11-18T18:20:34,611 INFO [RS_OPEN_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1. 2024-11-18T18:20:34,612 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=12222987219b43598fd47740805cf6b1, regionState=OPEN, openSeqNum=2, regionLocation=961a81da9827,38009,1731953921685 2024-11-18T18:20:34,618 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12222987219b43598fd47740805cf6b1, server=961a81da9827,38009,1731953921685 because future has completed 2024-11-18T18:20:34,627 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-18T18:20:34,627 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 12222987219b43598fd47740805cf6b1, server=961a81da9827,38009,1731953921685 in 248 msec 2024-11-18T18:20:34,647 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T18:20:34,647 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731954034647"}]},"ts":"1731954034647"} 2024-11-18T18:20:34,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-18T18:20:34,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=12222987219b43598fd47740805cf6b1, ASSIGN in 415 msec 2024-11-18T18:20:34,651 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-18T18:20:34,653 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T18:20:34,656 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 511 msec 2024-11-18T18:20:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39417 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-18T18:20:35,171 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-18T18:20:35,173 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-18T18:20:35,173 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:20:35,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-18T18:20:35,183 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:20:35,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-18T18:20:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1., hostname=961a81da9827,38009,1731953921685, seqNum=2] 2024-11-18T18:20:35,225 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-18T18:20:35,225 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-18T18:20:35,232 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-18T18:20:35,235 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-18T18:20:36,243 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1., hostname=961a81da9827,38009,1731953921685, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1., hostname=961a81da9827,38009,1731953921685, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=961a81da9827:38009 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms 2024-11-18T18:20:36,244 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1., hostname=961a81da9827,38009,1731953921685, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms 2024-11-18T18:20:36,244 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1., hostname=961a81da9827,38009,1731953921685, seqNum=2 from cache 2024-11-18T18:20:36,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-11-18T18:20:37,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1., hostname=961a81da9827,38009,1731953921685, seqNum=2] 2024-11-18T18:20:37,270 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38009 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-18T18:20:38,242 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1., hostname=961a81da9827,38009,1731953921685, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1., hostname=961a81da9827,38009,1731953921685, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=961a81da9827:38009 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=974ms, rpcTimeout=966ms 2024-11-18T18:20:38,242 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1., hostname=961a81da9827,38009,1731953921685, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=974ms, rpcTimeout=966ms 2024-11-18T18:20:38,242 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1., hostname=961a81da9827,38009,1731953921685, seqNum=2 from cache 2024-11-18T18:20:38,278 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-838902606_22 at /127.0.0.1:53736 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=784 (was 784), ProcessCount=11 (was 11), AvailableMemoryMB=1303 (was 1465) 2024-11-18T18:20:38,279 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T18:20:38,279 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T18:20:38,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:20:38,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:20:38,287 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T18:20:38,288 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T18:20:38,288 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T18:20:38,288 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T18:20:38,288 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:20:38,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:20:38,290 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T18:20:38,290 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1390163225, stopped=false 2024-11-18T18:20:38,290 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=961a81da9827,39417,1731953920254 2024-11-18T18:20:38,292 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T18:20:38,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38009-0x10107127afd0001, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T18:20:38,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38009-0x10107127afd0001, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:20:38,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T18:20:38,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:20:38,296 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38009-0x10107127afd0001, quorum=127.0.0.1:50225, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:20:38,297 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:20:38,301 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:20:38,304 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T18:20:38,305 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T18:20:38,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:20:38,306 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '961a81da9827,38009,1731953921685' ***** 2024-11-18T18:20:38,306 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T18:20:38,309 INFO [RS:0;961a81da9827:38009 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T18:20:38,310 INFO [RS:0;961a81da9827:38009 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T18:20:38,310 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T18:20:38,310 INFO [RS:0;961a81da9827:38009 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T18:20:38,310 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(3091): Received CLOSE for ca5265d377131ea04085fefa66db9922 2024-11-18T18:20:38,327 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(3091): Received CLOSE for 45e5d79edf6587776c61c37017185e82 2024-11-18T18:20:38,328 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(3091): Received CLOSE for 169a83608063e417ccb48f2d9f0ed8d3 2024-11-18T18:20:38,328 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(3091): Received CLOSE for c464a047be111daf460feab0aed90107 2024-11-18T18:20:38,328 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(3091): Received CLOSE for e1407279fa27e618cab5cc5f365ded5a 2024-11-18T18:20:38,328 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(3091): Received CLOSE for ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:38,328 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(3091): Received CLOSE for 19f5e2dceb3dffa2d39e146c4692d623 2024-11-18T18:20:38,328 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(3091): Received CLOSE for 12222987219b43598fd47740805cf6b1 2024-11-18T18:20:38,328 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(3091): Received CLOSE for 1ab6a75433d4cdfb37b6731cf70854ba 2024-11-18T18:20:38,328 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(3091): Received CLOSE for 0b55a66895220fe4244d54385a091298 2024-11-18T18:20:38,328 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(959): stopping server 961a81da9827,38009,1731953921685 2024-11-18T18:20:38,328 INFO [RS:0;961a81da9827:38009 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T18:20:38,328 INFO [RS:0;961a81da9827:38009 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;961a81da9827:38009. 2024-11-18T18:20:38,329 DEBUG [RS:0;961a81da9827:38009 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T18:20:38,329 DEBUG [RS:0;961a81da9827:38009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:20:38,329 INFO [RS:0;961a81da9827:38009 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T18:20:38,329 INFO [RS:0;961a81da9827:38009 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T18:20:38,329 INFO [RS:0;961a81da9827:38009 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T18:20:38,329 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T18:20:38,331 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ca5265d377131ea04085fefa66db9922, disabling compactions & flushes 2024-11-18T18:20:38,331 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922. 2024-11-18T18:20:38,331 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922. 2024-11-18T18:20:38,331 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922. after waiting 0 ms 2024-11-18T18:20:38,331 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922. 2024-11-18T18:20:38,337 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-18T18:20:38,338 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298, 12222987219b43598fd47740805cf6b1, 1588230740, 169a83608063e417ccb48f2d9f0ed8d3, 19f5e2dceb3dffa2d39e146c4692d623, 1ab6a75433d4cdfb37b6731cf70854ba, 45e5d79edf6587776c61c37017185e82, c464a047be111daf460feab0aed90107, ca5265d377131ea04085fefa66db9922, e1407279fa27e618cab5cc5f365ded5a, ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:38,338 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T18:20:38,338 INFO [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T18:20:38,338 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T18:20:38,338 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T18:20:38,338 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T18:20:38,354 INFO [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-18T18:20:38,357 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ca5265d377131ea04085fefa66db9922 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-18T18:20:38,538 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298, 12222987219b43598fd47740805cf6b1, 1588230740, 169a83608063e417ccb48f2d9f0ed8d3, 19f5e2dceb3dffa2d39e146c4692d623, 1ab6a75433d4cdfb37b6731cf70854ba, 45e5d79edf6587776c61c37017185e82, c464a047be111daf460feab0aed90107, ca5265d377131ea04085fefa66db9922, e1407279fa27e618cab5cc5f365ded5a, ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:38,550 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnOpenScanner/ca5265d377131ea04085fefa66db9922/.tmp/testFamily/af5e16e9a5af4c558ee0d9cbbf35b760 is 61, key is row-0/testFamily:testQualifier/1731953989173/Put/seqid=0 2024-11-18T18:20:38,589 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740/.tmp/info/88f4fc8f703c4b3fa1e88a9f4598b30a is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623./info:regioninfo/1731953987807/Put/seqid=0 2024-11-18T18:20:38,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741855_1031 (size=5049) 2024-11-18T18:20:38,625 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnOpenScanner/ca5265d377131ea04085fefa66db9922/.tmp/testFamily/af5e16e9a5af4c558ee0d9cbbf35b760 2024-11-18T18:20:38,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741856_1032 (size=20564) 2024-11-18T18:20:38,666 INFO [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740/.tmp/info/88f4fc8f703c4b3fa1e88a9f4598b30a 2024-11-18T18:20:38,738 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298, 12222987219b43598fd47740805cf6b1, 1588230740, 169a83608063e417ccb48f2d9f0ed8d3, 19f5e2dceb3dffa2d39e146c4692d623, 1ab6a75433d4cdfb37b6731cf70854ba, 45e5d79edf6587776c61c37017185e82, c464a047be111daf460feab0aed90107, ca5265d377131ea04085fefa66db9922, e1407279fa27e618cab5cc5f365ded5a, ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:38,762 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnOpenScanner/ca5265d377131ea04085fefa66db9922/.tmp/testFamily/af5e16e9a5af4c558ee0d9cbbf35b760 as hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnOpenScanner/ca5265d377131ea04085fefa66db9922/testFamily/af5e16e9a5af4c558ee0d9cbbf35b760 2024-11-18T18:20:38,771 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-18T18:20:38,823 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnOpenScanner/ca5265d377131ea04085fefa66db9922/testFamily/af5e16e9a5af4c558ee0d9cbbf35b760, entries=4, sequenceid=8, filesize=4.9 K 2024-11-18T18:20:38,844 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ca5265d377131ea04085fefa66db9922 in 505ms, sequenceid=8, compaction requested=false 2024-11-18T18:20:38,853 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740/.tmp/ns/ad03e53c74344171a16323ed31c96b41 is 43, key is default/ns:d/1731953986595/Put/seqid=0 2024-11-18T18:20:38,887 INFO [regionserver/961a81da9827:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-18T18:20:38,887 INFO [regionserver/961a81da9827:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-18T18:20:38,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741857_1033 (size=5153) 2024-11-18T18:20:38,913 INFO [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740/.tmp/ns/ad03e53c74344171a16323ed31c96b41 2024-11-18T18:20:38,927 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnOpenScanner/ca5265d377131ea04085fefa66db9922/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-18T18:20:38,936 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922. 2024-11-18T18:20:38,936 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ca5265d377131ea04085fefa66db9922: Waiting for close lock at 1731954038330Running coprocessor pre-close hooks at 1731954038331 (+1 ms)Disabling compacts and flushes for region at 1731954038331Disabling writes for close at 1731954038331Obtaining lock to block concurrent updates at 1731954038358 (+27 ms)Preparing flush snapshotting stores in ca5265d377131ea04085fefa66db9922 at 1731954038358Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731954038382 (+24 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922. at 1731954038385 (+3 ms)Flushing ca5265d377131ea04085fefa66db9922/testFamily: creating writer at 1731954038387 (+2 ms)Flushing ca5265d377131ea04085fefa66db9922/testFamily: appending metadata at 1731954038536 (+149 ms)Flushing ca5265d377131ea04085fefa66db9922/testFamily: closing flushed file at 1731954038540 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@571dea95: reopening flushed file at 1731954038751 (+211 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ca5265d377131ea04085fefa66db9922 in 505ms, sequenceid=8, compaction requested=false at 1731954038844 (+93 ms)Writing region close event to WAL at 1731954038867 (+23 ms)Running coprocessor post-close hooks at 1731954038929 (+62 ms)Closed at 1731954038936 (+7 ms) 2024-11-18T18:20:38,937 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731953988139.ca5265d377131ea04085fefa66db9922. 2024-11-18T18:20:38,937 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 45e5d79edf6587776c61c37017185e82, disabling compactions & flushes 2024-11-18T18:20:38,937 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82. 2024-11-18T18:20:38,937 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82. 2024-11-18T18:20:38,937 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82. after waiting 0 ms 2024-11-18T18:20:38,937 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82. 2024-11-18T18:20:38,937 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 45e5d79edf6587776c61c37017185e82 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-18T18:20:38,939 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298, 12222987219b43598fd47740805cf6b1, 1588230740, 169a83608063e417ccb48f2d9f0ed8d3, 19f5e2dceb3dffa2d39e146c4692d623, 1ab6a75433d4cdfb37b6731cf70854ba, 45e5d79edf6587776c61c37017185e82, c464a047be111daf460feab0aed90107, e1407279fa27e618cab5cc5f365ded5a, ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:38,976 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnNext/45e5d79edf6587776c61c37017185e82/.tmp/testFamily/48788515603c471187021df261b2e3ce is 61, key is row-0/testFamily:testQualifier/1731954014746/Put/seqid=0 2024-11-18T18:20:38,984 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740/.tmp/table/4126d7f688aa426a85aff3a4a1f39d55 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1731953987822/Put/seqid=0 2024-11-18T18:20:39,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741858_1034 (size=5049) 2024-11-18T18:20:39,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741859_1035 (size=6699) 2024-11-18T18:20:39,139 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298, 12222987219b43598fd47740805cf6b1, 1588230740, 169a83608063e417ccb48f2d9f0ed8d3, 19f5e2dceb3dffa2d39e146c4692d623, 1ab6a75433d4cdfb37b6731cf70854ba, 45e5d79edf6587776c61c37017185e82, c464a047be111daf460feab0aed90107, e1407279fa27e618cab5cc5f365ded5a, ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:39,340 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-11-18T18:20:39,340 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298, 12222987219b43598fd47740805cf6b1, 1588230740, 169a83608063e417ccb48f2d9f0ed8d3, 19f5e2dceb3dffa2d39e146c4692d623, 1ab6a75433d4cdfb37b6731cf70854ba, 45e5d79edf6587776c61c37017185e82, c464a047be111daf460feab0aed90107, e1407279fa27e618cab5cc5f365ded5a, ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:39,442 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnNext/45e5d79edf6587776c61c37017185e82/.tmp/testFamily/48788515603c471187021df261b2e3ce 2024-11-18T18:20:39,449 INFO [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740/.tmp/table/4126d7f688aa426a85aff3a4a1f39d55 2024-11-18T18:20:39,465 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnNext/45e5d79edf6587776c61c37017185e82/.tmp/testFamily/48788515603c471187021df261b2e3ce as hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnNext/45e5d79edf6587776c61c37017185e82/testFamily/48788515603c471187021df261b2e3ce 2024-11-18T18:20:39,469 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740/.tmp/info/88f4fc8f703c4b3fa1e88a9f4598b30a as hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740/info/88f4fc8f703c4b3fa1e88a9f4598b30a 2024-11-18T18:20:39,491 INFO [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740/info/88f4fc8f703c4b3fa1e88a9f4598b30a, entries=100, sequenceid=56, filesize=20.1 K 2024-11-18T18:20:39,491 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T18:20:39,493 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnNext/45e5d79edf6587776c61c37017185e82/testFamily/48788515603c471187021df261b2e3ce, entries=4, sequenceid=8, filesize=4.9 K 2024-11-18T18:20:39,494 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740/.tmp/ns/ad03e53c74344171a16323ed31c96b41 as hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740/ns/ad03e53c74344171a16323ed31c96b41 2024-11-18T18:20:39,495 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 45e5d79edf6587776c61c37017185e82 in 558ms, sequenceid=8, compaction requested=false 2024-11-18T18:20:39,507 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnNext/45e5d79edf6587776c61c37017185e82/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-18T18:20:39,512 INFO [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740/ns/ad03e53c74344171a16323ed31c96b41, entries=2, sequenceid=56, filesize=5.0 K 2024-11-18T18:20:39,513 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82. 2024-11-18T18:20:39,513 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 45e5d79edf6587776c61c37017185e82: Waiting for close lock at 1731954038937Running coprocessor pre-close hooks at 1731954038937Disabling compacts and flushes for region at 1731954038937Disabling writes for close at 1731954038937Obtaining lock to block concurrent updates at 1731954038937Preparing flush snapshotting stores in 45e5d79edf6587776c61c37017185e82 at 1731954038937Finished memstore snapshotting testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731954038938 (+1 ms)Flushing stores of testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82. at 1731954038939 (+1 ms)Flushing 45e5d79edf6587776c61c37017185e82/testFamily: creating writer at 1731954038939Flushing 45e5d79edf6587776c61c37017185e82/testFamily: appending metadata at 1731954038975 (+36 ms)Flushing 45e5d79edf6587776c61c37017185e82/testFamily: closing flushed file at 1731954038975Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b3a3325: reopening flushed file at 1731954039454 (+479 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 45e5d79edf6587776c61c37017185e82 in 558ms, sequenceid=8, compaction requested=false at 1731954039495 (+41 ms)Writing region close event to WAL at 1731954039496 (+1 ms)Running coprocessor post-close hooks at 1731954039512 (+16 ms)Closed at 1731954039512 2024-11-18T18:20:39,513 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1731954011694.45e5d79edf6587776c61c37017185e82. 2024-11-18T18:20:39,513 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 169a83608063e417ccb48f2d9f0ed8d3, disabling compactions & flushes 2024-11-18T18:20:39,513 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3. 2024-11-18T18:20:39,513 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3. 2024-11-18T18:20:39,513 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3. after waiting 0 ms 2024-11-18T18:20:39,513 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3. 2024-11-18T18:20:39,513 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 169a83608063e417ccb48f2d9f0ed8d3 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-18T18:20:39,515 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740/.tmp/table/4126d7f688aa426a85aff3a4a1f39d55 as hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740/table/4126d7f688aa426a85aff3a4a1f39d55 2024-11-18T18:20:39,540 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298, 12222987219b43598fd47740805cf6b1, 1588230740, 169a83608063e417ccb48f2d9f0ed8d3, 19f5e2dceb3dffa2d39e146c4692d623, 1ab6a75433d4cdfb37b6731cf70854ba, c464a047be111daf460feab0aed90107, e1407279fa27e618cab5cc5f365ded5a, ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:39,542 INFO [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740/table/4126d7f688aa426a85aff3a4a1f39d55, entries=20, sequenceid=56, filesize=6.5 K 2024-11-18T18:20:39,543 INFO [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1205ms, sequenceid=56, compaction requested=false 2024-11-18T18:20:39,567 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnNext/169a83608063e417ccb48f2d9f0ed8d3/.tmp/testFamily/43242a0ec7b24f9d88cc4feb015043f3 is 61, key is row-0/testFamily:testQualifier/1731954028068/Put/seqid=0 2024-11-18T18:20:39,598 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-18T18:20:39,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741860_1036 (size=5049) 2024-11-18T18:20:39,599 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T18:20:39,600 INFO [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T18:20:39,600 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731954038338Running coprocessor pre-close hooks at 1731954038338Disabling compacts and flushes for region at 1731954038338Disabling writes for close at 1731954038338Obtaining lock to block concurrent updates at 1731954038354 (+16 ms)Preparing flush snapshotting stores in 1588230740 at 1731954038354Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1731954038384 (+30 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731954038385 (+1 ms)Flushing 1588230740/info: creating writer at 1731954038387 (+2 ms)Flushing 1588230740/info: appending metadata at 1731954038578 (+191 ms)Flushing 1588230740/info: closing flushed file at 1731954038578Flushing 1588230740/ns: creating writer at 1731954038753 (+175 ms)Flushing 1588230740/ns: appending metadata at 1731954038850 (+97 ms)Flushing 1588230740/ns: closing flushed file at 1731954038850Flushing 1588230740/table: creating writer at 1731954038936 (+86 ms)Flushing 1588230740/table: appending metadata at 1731954038983 (+47 ms)Flushing 1588230740/table: closing flushed file at 1731954038983Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12c48fcc: reopening flushed file at 1731954039467 (+484 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37e81dc0: reopening flushed file at 1731954039491 (+24 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70d1b0c8: reopening flushed file at 1731954039512 (+21 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1205ms, sequenceid=56, compaction requested=false at 1731954039543 (+31 ms)Writing region close event to WAL at 1731954039557 (+14 ms)Running coprocessor post-close hooks at 1731954039599 (+42 ms)Closed at 1731954039600 (+1 ms) 2024-11-18T18:20:39,600 DEBUG [RS_CLOSE_META-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T18:20:39,741 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298, 12222987219b43598fd47740805cf6b1, 169a83608063e417ccb48f2d9f0ed8d3, 19f5e2dceb3dffa2d39e146c4692d623, 1ab6a75433d4cdfb37b6731cf70854ba, c464a047be111daf460feab0aed90107, e1407279fa27e618cab5cc5f365ded5a, ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:39,941 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298, 12222987219b43598fd47740805cf6b1, 169a83608063e417ccb48f2d9f0ed8d3, 19f5e2dceb3dffa2d39e146c4692d623, 1ab6a75433d4cdfb37b6731cf70854ba, c464a047be111daf460feab0aed90107, e1407279fa27e618cab5cc5f365ded5a, ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:40,002 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnNext/169a83608063e417ccb48f2d9f0ed8d3/.tmp/testFamily/43242a0ec7b24f9d88cc4feb015043f3 2024-11-18T18:20:40,021 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnNext/169a83608063e417ccb48f2d9f0ed8d3/.tmp/testFamily/43242a0ec7b24f9d88cc4feb015043f3 as hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnNext/169a83608063e417ccb48f2d9f0ed8d3/testFamily/43242a0ec7b24f9d88cc4feb015043f3 2024-11-18T18:20:40,033 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnNext/169a83608063e417ccb48f2d9f0ed8d3/testFamily/43242a0ec7b24f9d88cc4feb015043f3, entries=4, sequenceid=8, filesize=4.9 K 2024-11-18T18:20:40,035 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 169a83608063e417ccb48f2d9f0ed8d3 in 522ms, sequenceid=8, compaction requested=false 2024-11-18T18:20:40,056 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T18:20:40,067 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnNext/169a83608063e417ccb48f2d9f0ed8d3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-18T18:20:40,068 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3. 2024-11-18T18:20:40,069 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 169a83608063e417ccb48f2d9f0ed8d3: Waiting for close lock at 1731954039513Running coprocessor pre-close hooks at 1731954039513Disabling compacts and flushes for region at 1731954039513Disabling writes for close at 1731954039513Obtaining lock to block concurrent updates at 1731954039514 (+1 ms)Preparing flush snapshotting stores in 169a83608063e417ccb48f2d9f0ed8d3 at 1731954039514Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731954039514Flushing stores of hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3. at 1731954039520 (+6 ms)Flushing 169a83608063e417ccb48f2d9f0ed8d3/testFamily: creating writer at 1731954039520Flushing 169a83608063e417ccb48f2d9f0ed8d3/testFamily: appending metadata at 1731954039567 (+47 ms)Flushing 169a83608063e417ccb48f2d9f0ed8d3/testFamily: closing flushed file at 1731954039567Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@274ee4a: reopening flushed file at 1731954040018 (+451 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 169a83608063e417ccb48f2d9f0ed8d3 in 522ms, sequenceid=8, compaction requested=false at 1731954040035 (+17 ms)Writing region close event to WAL at 1731954040042 (+7 ms)Running coprocessor post-close hooks at 1731954040068 (+26 ms)Closed at 1731954040068 2024-11-18T18:20:40,069 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1731954027011.169a83608063e417ccb48f2d9f0ed8d3. 2024-11-18T18:20:40,069 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c464a047be111daf460feab0aed90107, disabling compactions & flushes 2024-11-18T18:20:40,069 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107. 2024-11-18T18:20:40,069 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107. 2024-11-18T18:20:40,069 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107. after waiting 0 ms 2024-11-18T18:20:40,069 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107. 2024-11-18T18:20:40,069 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c464a047be111daf460feab0aed90107 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-18T18:20:40,127 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/c464a047be111daf460feab0aed90107/.tmp/testFamily/ffba79a7374d45759cb09d11cd5a57db is 61, key is row-0/testFamily:testQualifier/1731954020918/Put/seqid=0 2024-11-18T18:20:40,140 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-18T18:20:40,141 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298, 12222987219b43598fd47740805cf6b1, 19f5e2dceb3dffa2d39e146c4692d623, 1ab6a75433d4cdfb37b6731cf70854ba, c464a047be111daf460feab0aed90107, e1407279fa27e618cab5cc5f365ded5a, ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:40,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741861_1037 (size=5049) 2024-11-18T18:20:40,342 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1321): Waiting on 7 regions to close 2024-11-18T18:20:40,342 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1325): Online Regions={c464a047be111daf460feab0aed90107=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107., e1407279fa27e618cab5cc5f365ded5a=testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a., ed0869c8914ca2d0e3dc8322a35c1663=testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663., 19f5e2dceb3dffa2d39e146c4692d623=testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623., 12222987219b43598fd47740805cf6b1=testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1., 1ab6a75433d4cdfb37b6731cf70854ba=hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba., 0b55a66895220fe4244d54385a091298=testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298.} 2024-11-18T18:20:40,342 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298, 12222987219b43598fd47740805cf6b1, 19f5e2dceb3dffa2d39e146c4692d623, 1ab6a75433d4cdfb37b6731cf70854ba, c464a047be111daf460feab0aed90107, e1407279fa27e618cab5cc5f365ded5a, ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:40,542 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298, 12222987219b43598fd47740805cf6b1, 19f5e2dceb3dffa2d39e146c4692d623, 1ab6a75433d4cdfb37b6731cf70854ba, c464a047be111daf460feab0aed90107, e1407279fa27e618cab5cc5f365ded5a, ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:40,558 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/c464a047be111daf460feab0aed90107/.tmp/testFamily/ffba79a7374d45759cb09d11cd5a57db 2024-11-18T18:20:40,578 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/c464a047be111daf460feab0aed90107/.tmp/testFamily/ffba79a7374d45759cb09d11cd5a57db as hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/c464a047be111daf460feab0aed90107/testFamily/ffba79a7374d45759cb09d11cd5a57db 2024-11-18T18:20:40,590 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/c464a047be111daf460feab0aed90107/testFamily/ffba79a7374d45759cb09d11cd5a57db, entries=4, sequenceid=8, filesize=4.9 K 2024-11-18T18:20:40,593 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c464a047be111daf460feab0aed90107 in 524ms, sequenceid=8, compaction requested=false 2024-11-18T18:20:40,653 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/c464a047be111daf460feab0aed90107/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-18T18:20:40,655 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107. 2024-11-18T18:20:40,655 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c464a047be111daf460feab0aed90107: Waiting for close lock at 1731954040069Running coprocessor pre-close hooks at 1731954040069Disabling compacts and flushes for region at 1731954040069Disabling writes for close at 1731954040069Obtaining lock to block concurrent updates at 1731954040069Preparing flush snapshotting stores in c464a047be111daf460feab0aed90107 at 1731954040069Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731954040070 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107. at 1731954040082 (+12 ms)Flushing c464a047be111daf460feab0aed90107/testFamily: creating writer at 1731954040082Flushing c464a047be111daf460feab0aed90107/testFamily: appending metadata at 1731954040126 (+44 ms)Flushing c464a047be111daf460feab0aed90107/testFamily: closing flushed file at 1731954040127 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@523dc5c9: reopening flushed file at 1731954040576 (+449 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c464a047be111daf460feab0aed90107 in 524ms, sequenceid=8, compaction requested=false at 1731954040593 (+17 ms)Writing region close event to WAL at 1731954040612 (+19 ms)Running coprocessor post-close hooks at 1731954040655 (+43 ms)Closed at 1731954040655 2024-11-18T18:20:40,655 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731954017838.c464a047be111daf460feab0aed90107. 2024-11-18T18:20:40,655 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e1407279fa27e618cab5cc5f365ded5a, disabling compactions & flushes 2024-11-18T18:20:40,656 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a. 2024-11-18T18:20:40,656 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a. 2024-11-18T18:20:40,656 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a. after waiting 0 ms 2024-11-18T18:20:40,656 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a. 2024-11-18T18:20:40,656 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e1407279fa27e618cab5cc5f365ded5a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-18T18:20:40,704 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnOpenScanner/e1407279fa27e618cab5cc5f365ded5a/.tmp/testFamily/22d5da6f44e143368e469344cf20f528 is 61, key is row-0/testFamily:testQualifier/1731954004540/Put/seqid=0 2024-11-18T18:20:40,743 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298, 12222987219b43598fd47740805cf6b1, 19f5e2dceb3dffa2d39e146c4692d623, 1ab6a75433d4cdfb37b6731cf70854ba, e1407279fa27e618cab5cc5f365ded5a, ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:40,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741862_1038 (size=5049) 2024-11-18T18:20:40,747 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnOpenScanner/e1407279fa27e618cab5cc5f365ded5a/.tmp/testFamily/22d5da6f44e143368e469344cf20f528 2024-11-18T18:20:40,762 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnOpenScanner/e1407279fa27e618cab5cc5f365ded5a/.tmp/testFamily/22d5da6f44e143368e469344cf20f528 as hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnOpenScanner/e1407279fa27e618cab5cc5f365ded5a/testFamily/22d5da6f44e143368e469344cf20f528 2024-11-18T18:20:40,775 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnOpenScanner/e1407279fa27e618cab5cc5f365ded5a/testFamily/22d5da6f44e143368e469344cf20f528, entries=4, sequenceid=8, filesize=4.9 K 2024-11-18T18:20:40,779 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e1407279fa27e618cab5cc5f365ded5a in 123ms, sequenceid=8, compaction requested=false 2024-11-18T18:20:40,813 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnOpenScanner/e1407279fa27e618cab5cc5f365ded5a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-18T18:20:40,822 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a. 2024-11-18T18:20:40,822 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e1407279fa27e618cab5cc5f365ded5a: Waiting for close lock at 1731954040655Running coprocessor pre-close hooks at 1731954040655Disabling compacts and flushes for region at 1731954040655Disabling writes for close at 1731954040656 (+1 ms)Obtaining lock to block concurrent updates at 1731954040656Preparing flush snapshotting stores in e1407279fa27e618cab5cc5f365ded5a at 1731954040656Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731954040656Flushing stores of testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a. at 1731954040657 (+1 ms)Flushing e1407279fa27e618cab5cc5f365ded5a/testFamily: creating writer at 1731954040658 (+1 ms)Flushing e1407279fa27e618cab5cc5f365ded5a/testFamily: appending metadata at 1731954040703 (+45 ms)Flushing e1407279fa27e618cab5cc5f365ded5a/testFamily: closing flushed file at 1731954040703Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7eec404e: reopening flushed file at 1731954040760 (+57 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e1407279fa27e618cab5cc5f365ded5a in 123ms, sequenceid=8, compaction requested=false at 1731954040779 (+19 ms)Writing region close event to WAL at 1731954040787 (+8 ms)Running coprocessor post-close hooks at 1731954040821 (+34 ms)Closed at 1731954040822 (+1 ms) 2024-11-18T18:20:40,822 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1731954003501.e1407279fa27e618cab5cc5f365ded5a. 2024-11-18T18:20:40,822 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ed0869c8914ca2d0e3dc8322a35c1663, disabling compactions & flushes 2024-11-18T18:20:40,822 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663. 2024-11-18T18:20:40,823 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663. 2024-11-18T18:20:40,823 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663. after waiting 0 ms 2024-11-18T18:20:40,823 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663. 2024-11-18T18:20:40,823 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ed0869c8914ca2d0e3dc8322a35c1663 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-18T18:20:40,874 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnNextAsync/ed0869c8914ca2d0e3dc8322a35c1663/.tmp/testFamily/033a921b442c4381a4d0f3f3021f0c4e is 61, key is row-0/testFamily:testQualifier/1731954008630/Put/seqid=0 2024-11-18T18:20:40,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741863_1039 (size=5049) 2024-11-18T18:20:40,905 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnNextAsync/ed0869c8914ca2d0e3dc8322a35c1663/.tmp/testFamily/033a921b442c4381a4d0f3f3021f0c4e 2024-11-18T18:20:40,943 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298, 12222987219b43598fd47740805cf6b1, 19f5e2dceb3dffa2d39e146c4692d623, 1ab6a75433d4cdfb37b6731cf70854ba, ed0869c8914ca2d0e3dc8322a35c1663 2024-11-18T18:20:40,957 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnNextAsync/ed0869c8914ca2d0e3dc8322a35c1663/.tmp/testFamily/033a921b442c4381a4d0f3f3021f0c4e as hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnNextAsync/ed0869c8914ca2d0e3dc8322a35c1663/testFamily/033a921b442c4381a4d0f3f3021f0c4e 2024-11-18T18:20:40,959 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-18T18:20:40,959 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-18T18:20:40,961 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-18T18:20:40,961 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-18T18:20:40,961 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-18T18:20:40,961 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T18:20:40,961 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-18T18:20:40,961 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-18T18:20:40,961 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-18T18:20:40,983 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnNextAsync/ed0869c8914ca2d0e3dc8322a35c1663/testFamily/033a921b442c4381a4d0f3f3021f0c4e, entries=4, sequenceid=8, filesize=4.9 K 2024-11-18T18:20:40,988 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ed0869c8914ca2d0e3dc8322a35c1663 in 165ms, sequenceid=8, compaction requested=false 2024-11-18T18:20:41,018 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnNextAsync/ed0869c8914ca2d0e3dc8322a35c1663/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-18T18:20:41,019 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663. 2024-11-18T18:20:41,019 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ed0869c8914ca2d0e3dc8322a35c1663: Waiting for close lock at 1731954040822Running coprocessor pre-close hooks at 1731954040822Disabling compacts and flushes for region at 1731954040822Disabling writes for close at 1731954040823 (+1 ms)Obtaining lock to block concurrent updates at 1731954040823Preparing flush snapshotting stores in ed0869c8914ca2d0e3dc8322a35c1663 at 1731954040823Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731954040824 (+1 ms)Flushing stores of testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663. at 1731954040830 (+6 ms)Flushing ed0869c8914ca2d0e3dc8322a35c1663/testFamily: creating writer at 1731954040830Flushing ed0869c8914ca2d0e3dc8322a35c1663/testFamily: appending metadata at 1731954040873 (+43 ms)Flushing ed0869c8914ca2d0e3dc8322a35c1663/testFamily: closing flushed file at 1731954040873Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@273c5816: reopening flushed file at 1731954040950 (+77 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ed0869c8914ca2d0e3dc8322a35c1663 in 165ms, sequenceid=8, compaction requested=false at 1731954040988 (+38 ms)Writing region close event to WAL at 1731954041006 (+18 ms)Running coprocessor post-close hooks at 1731954041019 (+13 ms)Closed at 1731954041019 2024-11-18T18:20:41,019 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1731954007592.ed0869c8914ca2d0e3dc8322a35c1663. 2024-11-18T18:20:41,019 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 19f5e2dceb3dffa2d39e146c4692d623, disabling compactions & flushes 2024-11-18T18:20:41,020 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623. 2024-11-18T18:20:41,020 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623. 2024-11-18T18:20:41,020 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623. after waiting 0 ms 2024-11-18T18:20:41,020 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623. 2024-11-18T18:20:41,020 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 19f5e2dceb3dffa2d39e146c4692d623 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-18T18:20:41,072 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testRetryOutOfOrderScannerNextExceptionAsync/19f5e2dceb3dffa2d39e146c4692d623/.tmp/testFamily/088e061f96e44428ac282cde6e7f0f8a is 61, key is row-0/testFamily:testQualifier/1731953988046/Put/seqid=0 2024-11-18T18:20:41,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741864_1040 (size=5049) 2024-11-18T18:20:41,144 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298, 12222987219b43598fd47740805cf6b1, 19f5e2dceb3dffa2d39e146c4692d623, 1ab6a75433d4cdfb37b6731cf70854ba 2024-11-18T18:20:41,349 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-18T18:20:41,349 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1325): Online Regions={19f5e2dceb3dffa2d39e146c4692d623=testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623., 12222987219b43598fd47740805cf6b1=testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1., 1ab6a75433d4cdfb37b6731cf70854ba=hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba., 0b55a66895220fe4244d54385a091298=testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298.} 2024-11-18T18:20:41,349 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298, 12222987219b43598fd47740805cf6b1, 19f5e2dceb3dffa2d39e146c4692d623, 1ab6a75433d4cdfb37b6731cf70854ba 2024-11-18T18:20:41,506 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testRetryOutOfOrderScannerNextExceptionAsync/19f5e2dceb3dffa2d39e146c4692d623/.tmp/testFamily/088e061f96e44428ac282cde6e7f0f8a 2024-11-18T18:20:41,526 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testRetryOutOfOrderScannerNextExceptionAsync/19f5e2dceb3dffa2d39e146c4692d623/.tmp/testFamily/088e061f96e44428ac282cde6e7f0f8a as hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testRetryOutOfOrderScannerNextExceptionAsync/19f5e2dceb3dffa2d39e146c4692d623/testFamily/088e061f96e44428ac282cde6e7f0f8a 2024-11-18T18:20:41,538 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testRetryOutOfOrderScannerNextExceptionAsync/19f5e2dceb3dffa2d39e146c4692d623/testFamily/088e061f96e44428ac282cde6e7f0f8a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-18T18:20:41,539 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 19f5e2dceb3dffa2d39e146c4692d623 in 519ms, sequenceid=8, compaction requested=false 2024-11-18T18:20:41,550 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298, 12222987219b43598fd47740805cf6b1, 19f5e2dceb3dffa2d39e146c4692d623, 1ab6a75433d4cdfb37b6731cf70854ba 2024-11-18T18:20:41,568 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testRetryOutOfOrderScannerNextExceptionAsync/19f5e2dceb3dffa2d39e146c4692d623/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-18T18:20:41,571 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623. 2024-11-18T18:20:41,571 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 19f5e2dceb3dffa2d39e146c4692d623: Waiting for close lock at 1731954041019Running coprocessor pre-close hooks at 1731954041019Disabling compacts and flushes for region at 1731954041019Disabling writes for close at 1731954041020 (+1 ms)Obtaining lock to block concurrent updates at 1731954041020Preparing flush snapshotting stores in 19f5e2dceb3dffa2d39e146c4692d623 at 1731954041020Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731954041020Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623. at 1731954041021 (+1 ms)Flushing 19f5e2dceb3dffa2d39e146c4692d623/testFamily: creating writer at 1731954041022 (+1 ms)Flushing 19f5e2dceb3dffa2d39e146c4692d623/testFamily: appending metadata at 1731954041072 (+50 ms)Flushing 19f5e2dceb3dffa2d39e146c4692d623/testFamily: closing flushed file at 1731954041072Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54d716de: reopening flushed file at 1731954041521 (+449 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 19f5e2dceb3dffa2d39e146c4692d623 in 519ms, sequenceid=8, compaction requested=false at 1731954041539 (+18 ms)Writing region close event to WAL at 1731954041540 (+1 ms)Running coprocessor post-close hooks at 1731954041571 (+31 ms)Closed at 1731954041571 2024-11-18T18:20:41,572 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731953986968.19f5e2dceb3dffa2d39e146c4692d623. 2024-11-18T18:20:41,572 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 12222987219b43598fd47740805cf6b1, disabling compactions & flushes 2024-11-18T18:20:41,572 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1. 2024-11-18T18:20:41,572 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1. 2024-11-18T18:20:41,572 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1. after waiting 0 ms 2024-11-18T18:20:41,572 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1. 2024-11-18T18:20:41,572 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 12222987219b43598fd47740805cf6b1 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-18T18:20:41,611 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnOpenScannerAsync/12222987219b43598fd47740805cf6b1/.tmp/testFamily/eae146e2733e478090003ba8c0852ecb is 61, key is row-0/testFamily:testQualifier/1731954035192/Put/seqid=0 2024-11-18T18:20:41,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741865_1041 (size=5049) 2024-11-18T18:20:41,655 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnOpenScannerAsync/12222987219b43598fd47740805cf6b1/.tmp/testFamily/eae146e2733e478090003ba8c0852ecb 2024-11-18T18:20:41,690 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnOpenScannerAsync/12222987219b43598fd47740805cf6b1/.tmp/testFamily/eae146e2733e478090003ba8c0852ecb as hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnOpenScannerAsync/12222987219b43598fd47740805cf6b1/testFamily/eae146e2733e478090003ba8c0852ecb 2024-11-18T18:20:41,706 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnOpenScannerAsync/12222987219b43598fd47740805cf6b1/testFamily/eae146e2733e478090003ba8c0852ecb, entries=4, sequenceid=8, filesize=4.9 K 2024-11-18T18:20:41,709 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 12222987219b43598fd47740805cf6b1 in 137ms, sequenceid=8, compaction requested=false 2024-11-18T18:20:41,737 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testNormalScanTimeoutOnOpenScannerAsync/12222987219b43598fd47740805cf6b1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-18T18:20:41,740 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1. 2024-11-18T18:20:41,740 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 12222987219b43598fd47740805cf6b1: Waiting for close lock at 1731954041572Running coprocessor pre-close hooks at 1731954041572Disabling compacts and flushes for region at 1731954041572Disabling writes for close at 1731954041572Obtaining lock to block concurrent updates at 1731954041572Preparing flush snapshotting stores in 12222987219b43598fd47740805cf6b1 at 1731954041572Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731954041572Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1. at 1731954041573 (+1 ms)Flushing 12222987219b43598fd47740805cf6b1/testFamily: creating writer at 1731954041577 (+4 ms)Flushing 12222987219b43598fd47740805cf6b1/testFamily: appending metadata at 1731954041610 (+33 ms)Flushing 12222987219b43598fd47740805cf6b1/testFamily: closing flushed file at 1731954041610Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d4df0a8: reopening flushed file at 1731954041688 (+78 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 12222987219b43598fd47740805cf6b1 in 137ms, sequenceid=8, compaction requested=false at 1731954041709 (+21 ms)Writing region close event to WAL at 1731954041717 (+8 ms)Running coprocessor post-close hooks at 1731954041740 (+23 ms)Closed at 1731954041740 2024-11-18T18:20:41,740 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731954034142.12222987219b43598fd47740805cf6b1. 2024-11-18T18:20:41,740 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1ab6a75433d4cdfb37b6731cf70854ba, disabling compactions & flushes 2024-11-18T18:20:41,741 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba. 2024-11-18T18:20:41,741 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba. 2024-11-18T18:20:41,741 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba. after waiting 0 ms 2024-11-18T18:20:41,741 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba. 2024-11-18T18:20:41,741 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1ab6a75433d4cdfb37b6731cf70854ba 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-18T18:20:41,752 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298, 1ab6a75433d4cdfb37b6731cf70854ba 2024-11-18T18:20:41,793 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnNextAsync/1ab6a75433d4cdfb37b6731cf70854ba/.tmp/testFamily/53c6bc3a54994acc946913121a938703 is 61, key is row-0/testFamily:testQualifier/1731953996313/Put/seqid=0 2024-11-18T18:20:41,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741866_1042 (size=5049) 2024-11-18T18:20:41,952 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298, 1ab6a75433d4cdfb37b6731cf70854ba 2024-11-18T18:20:42,153 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298, 1ab6a75433d4cdfb37b6731cf70854ba 2024-11-18T18:20:42,217 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnNextAsync/1ab6a75433d4cdfb37b6731cf70854ba/.tmp/testFamily/53c6bc3a54994acc946913121a938703 2024-11-18T18:20:42,240 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnNextAsync/1ab6a75433d4cdfb37b6731cf70854ba/.tmp/testFamily/53c6bc3a54994acc946913121a938703 as hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnNextAsync/1ab6a75433d4cdfb37b6731cf70854ba/testFamily/53c6bc3a54994acc946913121a938703 2024-11-18T18:20:42,251 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnNextAsync/1ab6a75433d4cdfb37b6731cf70854ba/testFamily/53c6bc3a54994acc946913121a938703, entries=4, sequenceid=8, filesize=4.9 K 2024-11-18T18:20:42,253 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1ab6a75433d4cdfb37b6731cf70854ba in 512ms, sequenceid=8, compaction requested=false 2024-11-18T18:20:42,271 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/hbase/testMetaScanTimeoutOnNextAsync/1ab6a75433d4cdfb37b6731cf70854ba/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-18T18:20:42,272 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba. 2024-11-18T18:20:42,272 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1ab6a75433d4cdfb37b6731cf70854ba: Waiting for close lock at 1731954041740Running coprocessor pre-close hooks at 1731954041740Disabling compacts and flushes for region at 1731954041740Disabling writes for close at 1731954041741 (+1 ms)Obtaining lock to block concurrent updates at 1731954041741Preparing flush snapshotting stores in 1ab6a75433d4cdfb37b6731cf70854ba at 1731954041741Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731954041741Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba. at 1731954041742 (+1 ms)Flushing 1ab6a75433d4cdfb37b6731cf70854ba/testFamily: creating writer at 1731954041742Flushing 1ab6a75433d4cdfb37b6731cf70854ba/testFamily: appending metadata at 1731954041793 (+51 ms)Flushing 1ab6a75433d4cdfb37b6731cf70854ba/testFamily: closing flushed file at 1731954041793Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@609c111f: reopening flushed file at 1731954042236 (+443 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1ab6a75433d4cdfb37b6731cf70854ba in 512ms, sequenceid=8, compaction requested=false at 1731954042253 (+17 ms)Writing region close event to WAL at 1731954042262 (+9 ms)Running coprocessor post-close hooks at 1731954042272 (+10 ms)Closed at 1731954042272 2024-11-18T18:20:42,272 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731953995268.1ab6a75433d4cdfb37b6731cf70854ba. 2024-11-18T18:20:42,273 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0b55a66895220fe4244d54385a091298, disabling compactions & flushes 2024-11-18T18:20:42,273 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298. 2024-11-18T18:20:42,273 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298. 2024-11-18T18:20:42,273 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298. after waiting 0 ms 2024-11-18T18:20:42,273 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298. 2024-11-18T18:20:42,273 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0b55a66895220fe4244d54385a091298 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-18T18:20:42,308 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testRetryOutOfOrderScannerNextException/0b55a66895220fe4244d54385a091298/.tmp/testFamily/0c619c3fc4d8418eb6bb60ca42c69dcf is 61, key is row-0/testFamily:testQualifier/1731954003431/Put/seqid=0 2024-11-18T18:20:42,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741867_1043 (size=5049) 2024-11-18T18:20:42,340 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testRetryOutOfOrderScannerNextException/0b55a66895220fe4244d54385a091298/.tmp/testFamily/0c619c3fc4d8418eb6bb60ca42c69dcf 2024-11-18T18:20:42,353 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-18T18:20:42,353 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1325): Online Regions={0b55a66895220fe4244d54385a091298=testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298.} 2024-11-18T18:20:42,353 DEBUG [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1351): Waiting on 0b55a66895220fe4244d54385a091298 2024-11-18T18:20:42,364 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testRetryOutOfOrderScannerNextException/0b55a66895220fe4244d54385a091298/.tmp/testFamily/0c619c3fc4d8418eb6bb60ca42c69dcf as hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testRetryOutOfOrderScannerNextException/0b55a66895220fe4244d54385a091298/testFamily/0c619c3fc4d8418eb6bb60ca42c69dcf 2024-11-18T18:20:42,388 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testRetryOutOfOrderScannerNextException/0b55a66895220fe4244d54385a091298/testFamily/0c619c3fc4d8418eb6bb60ca42c69dcf, entries=4, sequenceid=8, filesize=4.9 K 2024-11-18T18:20:42,391 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0b55a66895220fe4244d54385a091298 in 118ms, sequenceid=8, compaction requested=false 2024-11-18T18:20:42,454 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/data/default/testRetryOutOfOrderScannerNextException/0b55a66895220fe4244d54385a091298/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-18T18:20:42,456 INFO [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298. 2024-11-18T18:20:42,456 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0b55a66895220fe4244d54385a091298: Waiting for close lock at 1731954042273Running coprocessor pre-close hooks at 1731954042273Disabling compacts and flushes for region at 1731954042273Disabling writes for close at 1731954042273Obtaining lock to block concurrent updates at 1731954042273Preparing flush snapshotting stores in 0b55a66895220fe4244d54385a091298 at 1731954042273Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731954042273Flushing stores of testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298. at 1731954042275 (+2 ms)Flushing 0b55a66895220fe4244d54385a091298/testFamily: creating writer at 1731954042275Flushing 0b55a66895220fe4244d54385a091298/testFamily: appending metadata at 1731954042307 (+32 ms)Flushing 0b55a66895220fe4244d54385a091298/testFamily: closing flushed file at 1731954042307Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@148b94f: reopening flushed file at 1731954042363 (+56 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0b55a66895220fe4244d54385a091298 in 118ms, sequenceid=8, compaction requested=false at 1731954042391 (+28 ms)Writing region close event to WAL at 1731954042412 (+21 ms)Running coprocessor post-close hooks at 1731954042456 (+44 ms)Closed at 1731954042456 2024-11-18T18:20:42,456 DEBUG [RS_CLOSE_REGION-regionserver/961a81da9827:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1731954002389.0b55a66895220fe4244d54385a091298. 2024-11-18T18:20:42,554 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(976): stopping server 961a81da9827,38009,1731953921685; all regions closed. 2024-11-18T18:20:42,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741834_1010 (size=19479) 2024-11-18T18:20:42,586 DEBUG [RS:0;961a81da9827:38009 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/oldWALs 2024-11-18T18:20:42,587 INFO [RS:0;961a81da9827:38009 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 961a81da9827%2C38009%2C1731953921685.meta:.meta(num 1731953986383) 2024-11-18T18:20:42,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741833_1009 (size=20031) 2024-11-18T18:20:42,612 DEBUG [RS:0;961a81da9827:38009 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/oldWALs 2024-11-18T18:20:42,612 INFO [RS:0;961a81da9827:38009 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 961a81da9827%2C38009%2C1731953921685:(num 1731953985956) 2024-11-18T18:20:42,612 DEBUG [RS:0;961a81da9827:38009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:20:42,612 INFO [RS:0;961a81da9827:38009 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T18:20:42,613 INFO [RS:0;961a81da9827:38009 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T18:20:42,613 INFO [RS:0;961a81da9827:38009 {}] hbase.ChoreService(370): Chore service for: regionserver/961a81da9827:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T18:20:42,614 INFO [RS:0;961a81da9827:38009 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T18:20:42,615 INFO [RS:0;961a81da9827:38009 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38009 2024-11-18T18:20:42,618 INFO [regionserver/961a81da9827:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T18:20:42,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T18:20:42,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38009-0x10107127afd0001, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/961a81da9827,38009,1731953921685 2024-11-18T18:20:42,629 INFO [RS:0;961a81da9827:38009 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T18:20:42,630 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [961a81da9827,38009,1731953921685] 2024-11-18T18:20:42,632 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/961a81da9827,38009,1731953921685 already deleted, retry=false 2024-11-18T18:20:42,633 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 961a81da9827,38009,1731953921685 expired; onlineServers=0 2024-11-18T18:20:42,633 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '961a81da9827,39417,1731953920254' ***** 2024-11-18T18:20:42,633 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-18T18:20:42,633 INFO [M:0;961a81da9827:39417 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T18:20:42,633 INFO [M:0;961a81da9827:39417 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T18:20:42,633 DEBUG [M:0;961a81da9827:39417 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T18:20:42,633 DEBUG [M:0;961a81da9827:39417 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T18:20:42,634 INFO [M:0;961a81da9827:39417 {}] hbase.ChoreService(370): Chore service for: master/961a81da9827:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T18:20:42,634 INFO [M:0;961a81da9827:39417 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T18:20:42,634 DEBUG [M:0;961a81da9827:39417 {}] master.HMaster(1795): Stopping service threads 2024-11-18T18:20:42,634 INFO [M:0;961a81da9827:39417 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T18:20:42,634 INFO [M:0;961a81da9827:39417 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:20:42,635 INFO [M:0;961a81da9827:39417 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T18:20:42,635 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T18:20:42,635 DEBUG [master/961a81da9827:0:becomeActiveMaster-HFileCleaner.large.0-1731953925732 {}] cleaner.HFileCleaner(306): Exit Thread[master/961a81da9827:0:becomeActiveMaster-HFileCleaner.large.0-1731953925732,5,FailOnTimeoutGroup] 2024-11-18T18:20:42,637 DEBUG [master/961a81da9827:0:becomeActiveMaster-HFileCleaner.small.0-1731953925735 {}] cleaner.HFileCleaner(306): Exit Thread[master/961a81da9827:0:becomeActiveMaster-HFileCleaner.small.0-1731953925735,5,FailOnTimeoutGroup] 2024-11-18T18:20:42,637 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T18:20:42,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T18:20:42,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:20:42,648 DEBUG [M:0;961a81da9827:39417 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-18T18:20:42,648 DEBUG [M:0;961a81da9827:39417 {}] master.ActiveMasterManager(353): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-18T18:20:42,650 INFO [M:0;961a81da9827:39417 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/.lastflushedseqids 2024-11-18T18:20:42,701 INFO [regionserver/961a81da9827:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T18:20:42,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741868_1044 (size=523) 2024-11-18T18:20:42,720 INFO [M:0;961a81da9827:39417 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T18:20:42,721 INFO [M:0;961a81da9827:39417 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T18:20:42,721 DEBUG [M:0;961a81da9827:39417 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T18:20:42,721 INFO [M:0;961a81da9827:39417 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:20:42,721 DEBUG [M:0;961a81da9827:39417 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:20:42,721 DEBUG [M:0;961a81da9827:39417 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T18:20:42,721 DEBUG [M:0;961a81da9827:39417 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:20:42,721 INFO [M:0;961a81da9827:39417 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=162.87 KB heapSize=190.48 KB 2024-11-18T18:20:42,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38009-0x10107127afd0001, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:20:42,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38009-0x10107127afd0001, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:20:42,733 INFO [RS:0;961a81da9827:38009 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T18:20:42,733 INFO [RS:0;961a81da9827:38009 {}] regionserver.HRegionServer(1031): Exiting; stopping=961a81da9827,38009,1731953921685; zookeeper connection closed. 2024-11-18T18:20:42,738 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@156c3689 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@156c3689 2024-11-18T18:20:42,738 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-18T18:20:42,782 DEBUG [M:0;961a81da9827:39417 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0b97a5494f50438197650658935b660e is 82, key is hbase:meta,,1/info:regioninfo/1731953986508/Put/seqid=0 2024-11-18T18:20:42,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741869_1045 (size=5672) 2024-11-18T18:20:42,824 INFO [M:0;961a81da9827:39417 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0b97a5494f50438197650658935b660e 2024-11-18T18:20:42,889 DEBUG [M:0;961a81da9827:39417 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5ad62b28a31f46bebbb5b21293410a41 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731953987830/Put/seqid=0 2024-11-18T18:20:42,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741870_1046 (size=14070) 2024-11-18T18:20:42,920 INFO [M:0;961a81da9827:39417 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.31 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5ad62b28a31f46bebbb5b21293410a41 2024-11-18T18:20:42,929 INFO [M:0;961a81da9827:39417 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5ad62b28a31f46bebbb5b21293410a41 2024-11-18T18:20:42,954 DEBUG [M:0;961a81da9827:39417 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/549d242441ca4e87b7cc067d7ce06e8b is 69, key is 961a81da9827,38009,1731953921685/rs:state/1731953955749/Put/seqid=0 2024-11-18T18:20:42,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741871_1047 (size=5156) 2024-11-18T18:20:43,378 INFO [M:0;961a81da9827:39417 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/549d242441ca4e87b7cc067d7ce06e8b 2024-11-18T18:20:43,405 DEBUG [M:0;961a81da9827:39417 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0b97a5494f50438197650658935b660e as hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0b97a5494f50438197650658935b660e 2024-11-18T18:20:43,416 INFO [M:0;961a81da9827:39417 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0b97a5494f50438197650658935b660e, entries=8, sequenceid=313, filesize=5.5 K 2024-11-18T18:20:43,418 DEBUG [M:0;961a81da9827:39417 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5ad62b28a31f46bebbb5b21293410a41 as hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5ad62b28a31f46bebbb5b21293410a41 2024-11-18T18:20:43,431 INFO [M:0;961a81da9827:39417 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5ad62b28a31f46bebbb5b21293410a41 2024-11-18T18:20:43,431 INFO [M:0;961a81da9827:39417 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5ad62b28a31f46bebbb5b21293410a41, entries=33, sequenceid=313, filesize=13.7 K 2024-11-18T18:20:43,433 DEBUG [M:0;961a81da9827:39417 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/549d242441ca4e87b7cc067d7ce06e8b as hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/549d242441ca4e87b7cc067d7ce06e8b 2024-11-18T18:20:43,451 INFO [M:0;961a81da9827:39417 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42805/user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/549d242441ca4e87b7cc067d7ce06e8b, entries=1, sequenceid=313, filesize=5.0 K 2024-11-18T18:20:43,458 INFO [M:0;961a81da9827:39417 {}] regionserver.HRegion(3140): Finished flush of dataSize ~162.87 KB/166775, heapSize ~190.19 KB/194752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 737ms, sequenceid=313, compaction requested=false 2024-11-18T18:20:43,486 INFO [M:0;961a81da9827:39417 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:20:43,486 DEBUG [M:0;961a81da9827:39417 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731954042721Disabling compacts and flushes for region at 1731954042721Disabling writes for close at 1731954042721Obtaining lock to block concurrent updates at 1731954042721Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731954042721Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166775, getHeapSize=194992, getOffHeapSize=0, getCellsCount=360 at 1731954042723 (+2 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731954042724 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731954042725 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731954042781 (+56 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731954042781Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731954042840 (+59 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731954042888 (+48 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731954042888Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731954042929 (+41 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731954042952 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731954042952Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@216a3ab3: reopening flushed file at 1731954043401 (+449 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c8be50f: reopening flushed file at 1731954043417 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f63a9fa: reopening flushed file at 1731954043432 (+15 ms)Finished flush of dataSize ~162.87 KB/166775, heapSize ~190.19 KB/194752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 737ms, sequenceid=313, compaction requested=false at 1731954043458 (+26 ms)Writing region close event to WAL at 1731954043486 (+28 ms)Closed at 1731954043486 2024-11-18T18:20:43,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41527 is added to blk_1073741830_1006 (size=187727) 2024-11-18T18:20:43,522 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/106bd4c3-58c5-bf69-ffde-b05ee0042a23/MasterData/WALs/961a81da9827,39417,1731953920254/961a81da9827%2C39417%2C1731953920254.1731953924281 not finished, retry = 0 2024-11-18T18:20:43,629 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T18:20:43,630 INFO [M:0;961a81da9827:39417 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T18:20:43,630 INFO [M:0;961a81da9827:39417 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39417 2024-11-18T18:20:43,630 INFO [M:0;961a81da9827:39417 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T18:20:43,736 INFO [M:0;961a81da9827:39417 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T18:20:43,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:20:43,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39417-0x10107127afd0000, quorum=127.0.0.1:50225, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:20:43,742 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1362c9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T18:20:43,745 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b8ee7e7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T18:20:43,745 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T18:20:43,745 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2be8c12a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T18:20:43,745 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4d71513b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/hadoop.log.dir/,STOPPED} 2024-11-18T18:20:43,770 WARN [BP-100917435-172.17.0.2-1731953913846 heartbeating to localhost/127.0.0.1:42805 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-18T18:20:43,770 WARN [BP-100917435-172.17.0.2-1731953913846 heartbeating to localhost/127.0.0.1:42805 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-100917435-172.17.0.2-1731953913846 (Datanode Uuid bd5f880e-bbd5-4322-b0c7-35c085034d3a) service to localhost/127.0.0.1:42805 2024-11-18T18:20:43,773 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/cluster_029d3cf4-b51c-5c2b-f2f7-b81fc26ce99d/data/data2/current/BP-100917435-172.17.0.2-1731953913846 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T18:20:43,773 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/cluster_029d3cf4-b51c-5c2b-f2f7-b81fc26ce99d/data/data1/current/BP-100917435-172.17.0.2-1731953913846 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T18:20:43,776 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-18T18:20:43,776 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-18T18:20:43,779 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-18T18:20:43,794 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@b5692ae{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T18:20:43,795 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1809bc91{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T18:20:43,795 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T18:20:43,795 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@549fd2eb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T18:20:43,795 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@24268d3a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b2012ca-46ab-b6c5-656f-d60a0548d7c8/hadoop.log.dir/,STOPPED} 2024-11-18T18:20:43,811 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T18:20:43,856 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down