2024-12-03 18:58:48,593 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-03 18:58:48,614 main DEBUG Took 0.018651 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 18:58:48,614 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 18:58:48,615 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 18:58:48,616 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 18:58:48,617 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:58:48,624 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 18:58:48,636 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:58:48,638 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:58:48,638 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:58:48,639 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:58:48,639 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:58:48,639 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:58:48,640 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:58:48,640 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:58:48,641 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:58:48,641 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:58:48,642 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:58:48,642 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:58:48,642 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:58:48,643 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:58:48,643 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:58:48,643 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:58:48,644 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:58:48,644 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:58:48,645 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:58:48,645 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:58:48,645 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:58:48,646 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:58:48,646 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:58:48,646 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 18:58:48,647 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:58:48,647 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 18:58:48,648 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 18:58:48,650 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 18:58:48,652 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 18:58:48,653 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 18:58:48,654 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 18:58:48,655 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 18:58:48,663 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 18:58:48,666 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 18:58:48,668 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 18:58:48,668 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 18:58:48,668 main DEBUG createAppenders(={Console}) 2024-12-03 18:58:48,669 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-12-03 18:58:48,670 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-03 18:58:48,670 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-12-03 18:58:48,670 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 18:58:48,671 main DEBUG OutputStream closed 2024-12-03 18:58:48,671 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 18:58:48,671 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 18:58:48,671 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-12-03 18:58:48,746 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 18:58:48,749 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 18:58:48,750 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 18:58:48,751 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 18:58:48,752 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 18:58:48,753 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 18:58:48,753 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 18:58:48,753 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 18:58:48,754 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 18:58:48,754 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 18:58:48,755 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 18:58:48,755 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 18:58:48,756 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 18:58:48,756 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 18:58:48,756 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 18:58:48,757 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 18:58:48,757 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 18:58:48,758 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 18:58:48,761 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 18:58:48,761 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-12-03 18:58:48,761 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 18:58:48,762 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-12-03T18:58:48,774 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-12-03 18:58:48,777 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 18:58:48,778 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T18:58:49,030 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702 2024-12-03T18:58:49,047 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T18:58:49,065 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/cluster_ede8b3be-827f-ea87-d65f-2a8e90336803, deleteOnExit=true 2024-12-03T18:58:49,066 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T18:58:49,067 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/test.cache.data in system properties and HBase conf 2024-12-03T18:58:49,068 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T18:58:49,069 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/hadoop.log.dir in system properties and HBase conf 2024-12-03T18:58:49,069 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T18:58:49,070 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T18:58:49,071 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T18:58:49,171 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T18:58:49,283 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T18:58:49,288 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T18:58:49,288 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T18:58:49,289 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T18:58:49,290 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T18:58:49,290 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T18:58:49,291 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T18:58:49,292 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T18:58:49,292 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T18:58:49,293 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T18:58:49,294 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/nfs.dump.dir in system properties and HBase conf 2024-12-03T18:58:49,294 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/java.io.tmpdir in system properties and HBase conf 2024-12-03T18:58:49,294 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T18:58:49,295 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T18:58:49,296 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T18:58:50,705 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T18:58:50,784 INFO [Time-limited test {}] log.Log(170): Logging initialized @2906ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T18:58:50,862 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T18:58:50,929 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T18:58:50,950 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T18:58:50,950 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T18:58:50,952 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T18:58:50,967 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T18:58:50,970 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/hadoop.log.dir/,AVAILABLE} 2024-12-03T18:58:50,971 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T18:58:51,172 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@eb30bc1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/java.io.tmpdir/jetty-localhost-37579-hadoop-hdfs-3_4_1-tests_jar-_-any-7500925995777809146/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T18:58:51,183 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:37579} 2024-12-03T18:58:51,184 INFO [Time-limited test {}] server.Server(415): Started @3306ms 2024-12-03T18:58:51,772 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T18:58:51,780 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T18:58:51,782 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T18:58:51,782 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T18:58:51,782 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T18:58:51,783 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/hadoop.log.dir/,AVAILABLE} 2024-12-03T18:58:51,784 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T18:58:51,899 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5976fb41{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/java.io.tmpdir/jetty-localhost-44387-hadoop-hdfs-3_4_1-tests_jar-_-any-15119788344276176739/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T18:58:51,900 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:44387} 2024-12-03T18:58:51,900 INFO [Time-limited test {}] server.Server(415): Started @4023ms 2024-12-03T18:58:51,956 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T18:58:52,986 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/cluster_ede8b3be-827f-ea87-d65f-2a8e90336803/data/data1/current/BP-43425679-172.17.0.2-1733252329908/current, will proceed with Du for space computation calculation, 2024-12-03T18:58:52,986 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/cluster_ede8b3be-827f-ea87-d65f-2a8e90336803/data/data2/current/BP-43425679-172.17.0.2-1733252329908/current, will proceed with Du for space computation calculation, 2024-12-03T18:58:53,015 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T18:58:53,073 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3b55da13dac7f271 with lease ID 0x45def04b12d12648: Processing first storage report for DS-66553b08-2ade-4967-b00a-55fa72770989 from datanode DatanodeRegistration(127.0.0.1:35847, datanodeUuid=6aad047e-1653-4b8b-85b9-787e7b8b647b, infoPort=36947, infoSecurePort=0, ipcPort=43899, storageInfo=lv=-57;cid=testClusterID;nsid=837015673;c=1733252329908) 2024-12-03T18:58:53,075 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3b55da13dac7f271 with lease ID 0x45def04b12d12648: from storage DS-66553b08-2ade-4967-b00a-55fa72770989 node DatanodeRegistration(127.0.0.1:35847, datanodeUuid=6aad047e-1653-4b8b-85b9-787e7b8b647b, infoPort=36947, infoSecurePort=0, ipcPort=43899, storageInfo=lv=-57;cid=testClusterID;nsid=837015673;c=1733252329908), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-03T18:58:53,075 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3b55da13dac7f271 with lease ID 0x45def04b12d12648: Processing first storage report for DS-fa40912e-9740-4f53-9296-3f1b75d98601 from datanode DatanodeRegistration(127.0.0.1:35847, datanodeUuid=6aad047e-1653-4b8b-85b9-787e7b8b647b, infoPort=36947, infoSecurePort=0, ipcPort=43899, storageInfo=lv=-57;cid=testClusterID;nsid=837015673;c=1733252329908) 2024-12-03T18:58:53,076 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3b55da13dac7f271 with lease ID 0x45def04b12d12648: from storage DS-fa40912e-9740-4f53-9296-3f1b75d98601 node DatanodeRegistration(127.0.0.1:35847, datanodeUuid=6aad047e-1653-4b8b-85b9-787e7b8b647b, infoPort=36947, infoSecurePort=0, ipcPort=43899, storageInfo=lv=-57;cid=testClusterID;nsid=837015673;c=1733252329908), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T18:58:53,124 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702 2024-12-03T18:58:53,209 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/cluster_ede8b3be-827f-ea87-d65f-2a8e90336803/zookeeper_0, clientPort=59911, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/cluster_ede8b3be-827f-ea87-d65f-2a8e90336803/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/cluster_ede8b3be-827f-ea87-d65f-2a8e90336803/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T18:58:53,222 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59911 2024-12-03T18:58:53,233 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T18:58:53,236 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T18:58:53,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741825_1001 (size=7) 2024-12-03T18:58:53,877 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e with version=8 2024-12-03T18:58:53,878 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/hbase-staging 2024-12-03T18:58:53,965 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T18:58:54,253 INFO [Time-limited test {}] client.ConnectionUtils(128): master/db5a5ccf5be8:0 server-side Connection retries=9 2024-12-03T18:58:54,264 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T18:58:54,264 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T18:58:54,270 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T18:58:54,270 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T18:58:54,270 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T18:58:54,408 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T18:58:54,474 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T18:58:54,486 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T18:58:54,491 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T18:58:54,525 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 101624 (auto-detected) 2024-12-03T18:58:54,526 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-03T18:58:54,551 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33603 2024-12-03T18:58:54,575 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33603 connecting to ZooKeeper ensemble=127.0.0.1:59911 2024-12-03T18:58:54,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:336030x0, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T18:58:54,709 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33603-0x1019c8fba300000 connected 2024-12-03T18:58:54,809 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T18:58:54,812 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T18:58:54,826 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T18:58:54,830 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e, hbase.cluster.distributed=false 2024-12-03T18:58:54,858 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T18:58:54,863 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33603 2024-12-03T18:58:54,863 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33603 2024-12-03T18:58:54,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33603 2024-12-03T18:58:54,868 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33603 2024-12-03T18:58:54,869 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33603 2024-12-03T18:58:54,979 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/db5a5ccf5be8:0 server-side Connection retries=9 2024-12-03T18:58:54,981 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T18:58:54,981 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T18:58:54,981 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T18:58:54,981 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T18:58:54,982 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T18:58:54,984 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T18:58:54,986 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T18:58:54,987 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32769 2024-12-03T18:58:54,989 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32769 connecting to ZooKeeper ensemble=127.0.0.1:59911 2024-12-03T18:58:54,990 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T18:58:54,993 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T18:58:55,013 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:327690x0, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T18:58:55,013 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:327690x0, quorum=127.0.0.1:59911, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T18:58:55,013 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32769-0x1019c8fba300001 connected 2024-12-03T18:58:55,018 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T18:58:55,025 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T18:58:55,028 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32769-0x1019c8fba300001, quorum=127.0.0.1:59911, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T18:58:55,034 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32769-0x1019c8fba300001, quorum=127.0.0.1:59911, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T18:58:55,034 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32769 2024-12-03T18:58:55,043 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32769 2024-12-03T18:58:55,044 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32769 2024-12-03T18:58:55,045 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32769 2024-12-03T18:58:55,045 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32769 2024-12-03T18:58:55,063 DEBUG [M:0;db5a5ccf5be8:33603 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;db5a5ccf5be8:33603 2024-12-03T18:58:55,064 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/db5a5ccf5be8,33603,1733252334072 2024-12-03T18:58:55,080 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32769-0x1019c8fba300001, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T18:58:55,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T18:58:55,083 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/db5a5ccf5be8,33603,1733252334072 2024-12-03T18:58:55,118 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32769-0x1019c8fba300001, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T18:58:55,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:58:55,118 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32769-0x1019c8fba300001, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:58:55,119 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T18:58:55,120 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/db5a5ccf5be8,33603,1733252334072 from backup master directory 2024-12-03T18:58:55,132 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32769-0x1019c8fba300001, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T18:58:55,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/db5a5ccf5be8,33603,1733252334072 2024-12-03T18:58:55,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T18:58:55,133 WARN [master/db5a5ccf5be8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T18:58:55,133 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=db5a5ccf5be8,33603,1733252334072 2024-12-03T18:58:55,137 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T18:58:55,138 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T18:58:55,198 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/hbase.id] with ID: 65bbdcb6-df76-46bf-9410-d71c06dd4d2e 2024-12-03T18:58:55,199 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/.tmp/hbase.id 2024-12-03T18:58:55,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741826_1002 (size=42) 2024-12-03T18:58:55,211 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/.tmp/hbase.id]:[hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/hbase.id] 2024-12-03T18:58:55,260 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T18:58:55,266 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T18:58:55,287 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-12-03T18:58:55,300 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32769-0x1019c8fba300001, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:58:55,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:58:55,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741827_1003 (size=196) 2024-12-03T18:58:55,336 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T18:58:55,339 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T18:58:55,358 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T18:58:55,362 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T18:58:55,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741828_1004 (size=1189) 2024-12-03T18:58:55,816 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/data/master/store 2024-12-03T18:58:55,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741829_1005 (size=34) 2024-12-03T18:58:55,843 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T18:58:55,846 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:58:55,847 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T18:58:55,847 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T18:58:55,848 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T18:58:55,849 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T18:58:55,849 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T18:58:55,850 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T18:58:55,851 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733252335847Disabling compacts and flushes for region at 1733252335847Disabling writes for close at 1733252335849 (+2 ms)Writing region close event to WAL at 1733252335850 (+1 ms)Closed at 1733252335850 2024-12-03T18:58:55,854 WARN [master/db5a5ccf5be8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/data/master/store/.initializing 2024-12-03T18:58:55,854 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/WALs/db5a5ccf5be8,33603,1733252334072 2024-12-03T18:58:55,863 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T18:58:55,879 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db5a5ccf5be8%2C33603%2C1733252334072, suffix=, logDir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/WALs/db5a5ccf5be8,33603,1733252334072, archiveDir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/oldWALs, maxLogs=10 2024-12-03T18:58:55,905 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/WALs/db5a5ccf5be8,33603,1733252334072/db5a5ccf5be8%2C33603%2C1733252334072.1733252335884, exclude list is [], retry=0 2024-12-03T18:58:55,922 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35847,DS-66553b08-2ade-4967-b00a-55fa72770989,DISK] 2024-12-03T18:58:55,925 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T18:58:55,962 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/WALs/db5a5ccf5be8,33603,1733252334072/db5a5ccf5be8%2C33603%2C1733252334072.1733252335884 2024-12-03T18:58:55,963 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36947:36947)] 2024-12-03T18:58:55,964 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T18:58:55,964 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:58:55,968 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:58:55,969 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:58:56,006 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:58:56,033 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T18:58:56,037 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:58:56,040 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T18:58:56,040 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:58:56,044 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T18:58:56,044 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:58:56,045 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T18:58:56,046 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:58:56,049 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T18:58:56,049 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:58:56,050 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T18:58:56,050 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:58:56,053 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T18:58:56,054 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:58:56,055 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T18:58:56,055 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:58:56,059 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:58:56,061 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:58:56,067 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:58:56,068 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:58:56,072 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T18:58:56,076 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T18:58:56,082 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T18:58:56,083 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72162602, jitterRate=0.07530656456947327}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T18:58:56,090 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733252335981Initializing all the Stores at 1733252335984 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733252335984Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733252335985 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733252335985Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733252335985Cleaning up temporary data from old regions at 1733252336068 (+83 ms)Region opened successfully at 1733252336090 (+22 ms) 2024-12-03T18:58:56,092 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T18:58:56,130 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681ddae2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db5a5ccf5be8/172.17.0.2:0 2024-12-03T18:58:56,163 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T18:58:56,176 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T18:58:56,176 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T18:58:56,179 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T18:58:56,181 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T18:58:56,186 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-03T18:58:56,187 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T18:58:56,217 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T18:58:56,228 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T18:58:56,311 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T18:58:56,314 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T18:58:56,316 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T18:58:56,328 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T18:58:56,331 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T18:58:56,343 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T18:58:56,353 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T18:58:56,355 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T18:58:56,363 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T18:58:56,381 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T18:58:56,391 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T18:58:56,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T18:58:56,406 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32769-0x1019c8fba300001, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T18:58:56,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:58:56,406 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32769-0x1019c8fba300001, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:58:56,409 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=db5a5ccf5be8,33603,1733252334072, sessionid=0x1019c8fba300000, setting cluster-up flag (Was=false) 2024-12-03T18:58:56,437 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32769-0x1019c8fba300001, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:58:56,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:58:56,469 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T18:58:56,472 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=db5a5ccf5be8,33603,1733252334072 2024-12-03T18:58:56,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:58:56,496 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32769-0x1019c8fba300001, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:58:56,528 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T18:58:56,531 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=db5a5ccf5be8,33603,1733252334072 2024-12-03T18:58:56,539 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T18:58:56,550 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(746): ClusterId : 65bbdcb6-df76-46bf-9410-d71c06dd4d2e 2024-12-03T18:58:56,553 DEBUG [RS:0;db5a5ccf5be8:32769 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T18:58:56,565 DEBUG [RS:0;db5a5ccf5be8:32769 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T18:58:56,566 DEBUG [RS:0;db5a5ccf5be8:32769 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T18:58:56,575 DEBUG [RS:0;db5a5ccf5be8:32769 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T18:58:56,576 DEBUG [RS:0;db5a5ccf5be8:32769 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22953b3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=db5a5ccf5be8/172.17.0.2:0 2024-12-03T18:58:56,592 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;db5a5ccf5be8:32769 2024-12-03T18:58:56,596 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T18:58:56,596 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T18:58:56,596 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T18:58:56,599 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(2659): reportForDuty to master=db5a5ccf5be8,33603,1733252334072 with port=32769, startcode=1733252334945 2024-12-03T18:58:56,613 DEBUG [RS:0;db5a5ccf5be8:32769 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T18:58:56,620 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T18:58:56,630 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T18:58:56,638 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T18:58:56,648 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53173, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T18:58:56,645 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: db5a5ccf5be8,33603,1733252334072 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T18:58:56,655 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/db5a5ccf5be8:0, corePoolSize=5, maxPoolSize=5 2024-12-03T18:58:56,655 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/db5a5ccf5be8:0, corePoolSize=5, maxPoolSize=5 2024-12-03T18:58:56,655 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/db5a5ccf5be8:0, corePoolSize=5, maxPoolSize=5 2024-12-03T18:58:56,656 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/db5a5ccf5be8:0, corePoolSize=5, maxPoolSize=5 2024-12-03T18:58:56,655 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33603 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T18:58:56,656 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/db5a5ccf5be8:0, corePoolSize=10, maxPoolSize=10 2024-12-03T18:58:56,656 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/db5a5ccf5be8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:58:56,656 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/db5a5ccf5be8:0, corePoolSize=2, maxPoolSize=2 2024-12-03T18:58:56,657 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/db5a5ccf5be8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:58:56,667 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T18:58:56,668 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T18:58:56,674 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:58:56,674 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T18:58:56,685 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733252366684 2024-12-03T18:58:56,685 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-03T18:58:56,686 WARN [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 30000 ms and then retrying. 2024-12-03T18:58:56,687 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T18:58:56,688 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T18:58:56,692 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T18:58:56,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741831_1007 (size=1321) 2024-12-03T18:58:56,692 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T18:58:56,693 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T18:58:56,693 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T18:58:56,694 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T18:58:56,698 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T18:58:56,699 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T18:58:56,699 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T18:58:56,701 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T18:58:56,702 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T18:58:56,703 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/db5a5ccf5be8:0:becomeActiveMaster-HFileCleaner.large.0-1733252336703,5,FailOnTimeoutGroup] 2024-12-03T18:58:56,704 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/db5a5ccf5be8:0:becomeActiveMaster-HFileCleaner.small.0-1733252336704,5,FailOnTimeoutGroup] 2024-12-03T18:58:56,704 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T18:58:56,705 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T18:58:56,706 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T18:58:56,706 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T18:58:57,096 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T18:58:57,097 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e 2024-12-03T18:58:57,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741832_1008 (size=32) 2024-12-03T18:58:57,510 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:58:57,513 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T18:58:57,516 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T18:58:57,516 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:58:57,518 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T18:58:57,518 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T18:58:57,521 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T18:58:57,521 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:58:57,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T18:58:57,523 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T18:58:57,525 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T18:58:57,525 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:58:57,526 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T18:58:57,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T18:58:57,529 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T18:58:57,529 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:58:57,530 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T18:58:57,531 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T18:58:57,532 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740 2024-12-03T18:58:57,533 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740 2024-12-03T18:58:57,537 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T18:58:57,537 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T18:58:57,538 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T18:58:57,541 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T18:58:57,545 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T18:58:57,546 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64009146, jitterRate=-0.04618939757347107}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T18:58:57,549 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733252337510Initializing all the Stores at 1733252337512 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733252337512Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733252337513 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733252337513Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733252337513Cleaning up temporary data from old regions at 1733252337537 (+24 ms)Region opened successfully at 1733252337549 (+12 ms) 2024-12-03T18:58:57,550 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T18:58:57,550 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T18:58:57,550 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T18:58:57,550 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T18:58:57,550 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T18:58:57,551 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T18:58:57,552 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733252337549Disabling compacts and flushes for region at 1733252337549Disabling writes for close at 1733252337550 (+1 ms)Writing region close event to WAL at 1733252337551 (+1 ms)Closed at 1733252337551 2024-12-03T18:58:57,556 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T18:58:57,556 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T18:58:57,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T18:58:57,572 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T18:58:57,575 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T18:58:57,726 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:58:58,727 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:58:59,728 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:00,729 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:01,729 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:02,730 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:03,731 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:04,471 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T18:59:04,471 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T18:59:04,731 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:05,732 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:06,733 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:07,733 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:08,734 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:09,735 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:10,736 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:11,736 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:12,737 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:13,738 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:14,738 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:15,739 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:16,740 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:17,740 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:18,741 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:19,742 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:20,742 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:21,743 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:22,744 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:23,120 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T18:59:23,745 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:24,745 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:25,746 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:26,687 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(2659): reportForDuty to master=db5a5ccf5be8,33603,1733252334072 with port=32769, startcode=1733252334945 2024-12-03T18:59:26,690 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33603 {}] master.ServerManager(363): Checking decommissioned status of RegionServer db5a5ccf5be8,32769,1733252334945 2024-12-03T18:59:26,692 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33603 {}] master.ServerManager(517): Registering regionserver=db5a5ccf5be8,32769,1733252334945 2024-12-03T18:59:26,700 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e 2024-12-03T18:59:26,700 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34585 2024-12-03T18:59:26,701 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T18:59:26,747 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:26,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T18:59:26,763 DEBUG [RS:0;db5a5ccf5be8:32769 {}] zookeeper.ZKUtil(111): regionserver:32769-0x1019c8fba300001, quorum=127.0.0.1:59911, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/db5a5ccf5be8,32769,1733252334945 2024-12-03T18:59:26,763 WARN [RS:0;db5a5ccf5be8:32769 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T18:59:26,763 INFO [RS:0;db5a5ccf5be8:32769 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T18:59:26,764 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/WALs/db5a5ccf5be8,32769,1733252334945 2024-12-03T18:59:26,766 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [db5a5ccf5be8,32769,1733252334945] 2024-12-03T18:59:26,791 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T18:59:26,802 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T18:59:26,808 INFO [RS:0;db5a5ccf5be8:32769 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T18:59:26,808 INFO [RS:0;db5a5ccf5be8:32769 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T18:59:26,809 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T18:59:26,815 INFO [RS:0;db5a5ccf5be8:32769 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T18:59:26,816 INFO [RS:0;db5a5ccf5be8:32769 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T18:59:26,817 DEBUG [RS:0;db5a5ccf5be8:32769 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/db5a5ccf5be8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:59:26,817 DEBUG [RS:0;db5a5ccf5be8:32769 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/db5a5ccf5be8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:59:26,817 DEBUG [RS:0;db5a5ccf5be8:32769 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:59:26,817 DEBUG [RS:0;db5a5ccf5be8:32769 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:59:26,817 DEBUG [RS:0;db5a5ccf5be8:32769 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/db5a5ccf5be8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:59:26,817 DEBUG [RS:0;db5a5ccf5be8:32769 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/db5a5ccf5be8:0, corePoolSize=2, maxPoolSize=2 2024-12-03T18:59:26,817 DEBUG [RS:0;db5a5ccf5be8:32769 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/db5a5ccf5be8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:59:26,818 DEBUG [RS:0;db5a5ccf5be8:32769 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/db5a5ccf5be8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:59:26,818 DEBUG [RS:0;db5a5ccf5be8:32769 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/db5a5ccf5be8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:59:26,818 DEBUG [RS:0;db5a5ccf5be8:32769 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/db5a5ccf5be8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:59:26,818 DEBUG [RS:0;db5a5ccf5be8:32769 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/db5a5ccf5be8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:59:26,818 DEBUG [RS:0;db5a5ccf5be8:32769 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/db5a5ccf5be8:0, corePoolSize=1, maxPoolSize=1 2024-12-03T18:59:26,818 DEBUG [RS:0;db5a5ccf5be8:32769 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/db5a5ccf5be8:0, corePoolSize=3, maxPoolSize=3 2024-12-03T18:59:26,818 DEBUG [RS:0;db5a5ccf5be8:32769 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/db5a5ccf5be8:0, corePoolSize=3, maxPoolSize=3 2024-12-03T18:59:26,820 INFO [RS:0;db5a5ccf5be8:32769 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T18:59:26,820 INFO [RS:0;db5a5ccf5be8:32769 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T18:59:26,820 INFO [RS:0;db5a5ccf5be8:32769 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T18:59:26,820 INFO [RS:0;db5a5ccf5be8:32769 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T18:59:26,820 INFO [RS:0;db5a5ccf5be8:32769 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T18:59:26,820 INFO [RS:0;db5a5ccf5be8:32769 {}] hbase.ChoreService(168): Chore ScheduledChore name=db5a5ccf5be8,32769,1733252334945-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T18:59:26,839 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T18:59:26,841 INFO [RS:0;db5a5ccf5be8:32769 {}] hbase.ChoreService(168): Chore ScheduledChore name=db5a5ccf5be8,32769,1733252334945-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T18:59:26,841 INFO [RS:0;db5a5ccf5be8:32769 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T18:59:26,841 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.Replication(171): db5a5ccf5be8,32769,1733252334945 started 2024-12-03T18:59:26,859 INFO [RS:0;db5a5ccf5be8:32769 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T18:59:26,860 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1482): Serving as db5a5ccf5be8,32769,1733252334945, RpcServer on db5a5ccf5be8/172.17.0.2:32769, sessionid=0x1019c8fba300001 2024-12-03T18:59:26,860 DEBUG [RS:0;db5a5ccf5be8:32769 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T18:59:26,861 DEBUG [RS:0;db5a5ccf5be8:32769 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager db5a5ccf5be8,32769,1733252334945 2024-12-03T18:59:26,861 DEBUG [RS:0;db5a5ccf5be8:32769 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db5a5ccf5be8,32769,1733252334945' 2024-12-03T18:59:26,861 DEBUG [RS:0;db5a5ccf5be8:32769 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T18:59:26,862 DEBUG [RS:0;db5a5ccf5be8:32769 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T18:59:26,863 DEBUG [RS:0;db5a5ccf5be8:32769 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T18:59:26,863 DEBUG [RS:0;db5a5ccf5be8:32769 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T18:59:26,863 DEBUG [RS:0;db5a5ccf5be8:32769 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager db5a5ccf5be8,32769,1733252334945 2024-12-03T18:59:26,863 DEBUG [RS:0;db5a5ccf5be8:32769 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'db5a5ccf5be8,32769,1733252334945' 2024-12-03T18:59:26,863 DEBUG [RS:0;db5a5ccf5be8:32769 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T18:59:26,864 DEBUG [RS:0;db5a5ccf5be8:32769 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T18:59:26,865 DEBUG [RS:0;db5a5ccf5be8:32769 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T18:59:26,865 INFO [RS:0;db5a5ccf5be8:32769 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T18:59:26,865 INFO [RS:0;db5a5ccf5be8:32769 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T18:59:27,747 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:28,748 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:29,749 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:30,750 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:31,750 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:32,751 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:33,752 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:34,752 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:35,753 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:36,754 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:37,755 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:38,755 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:39,756 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:40,757 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:41,757 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:42,758 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:43,759 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:44,759 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:45,760 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:46,761 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:47,762 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:48,762 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:49,763 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:50,764 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:51,764 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:52,765 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:53,120 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T18:59:53,766 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:54,766 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:55,767 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:56,768 WARN [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T18:59:56,873 INFO [RS:0;db5a5ccf5be8:32769 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T18:59:56,879 INFO [RS:0;db5a5ccf5be8:32769 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db5a5ccf5be8%2C32769%2C1733252334945, suffix=, logDir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/WALs/db5a5ccf5be8,32769,1733252334945, archiveDir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/oldWALs, maxLogs=32 2024-12-03T18:59:56,895 DEBUG [RS:0;db5a5ccf5be8:32769 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/WALs/db5a5ccf5be8,32769,1733252334945/db5a5ccf5be8%2C32769%2C1733252334945.1733252396881, exclude list is [], retry=0 2024-12-03T18:59:56,899 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35847,DS-66553b08-2ade-4967-b00a-55fa72770989,DISK] 2024-12-03T18:59:56,903 INFO [RS:0;db5a5ccf5be8:32769 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/WALs/db5a5ccf5be8,32769,1733252334945/db5a5ccf5be8%2C32769%2C1733252334945.1733252396881 2024-12-03T18:59:56,903 DEBUG [RS:0;db5a5ccf5be8:32769 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36947:36947)] 2024-12-03T18:59:57,022 DEBUG [db5a5ccf5be8:33603 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-03T18:59:57,036 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T18:59:57,042 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db5a5ccf5be8,32769,1733252334945, state=OPENING 2024-12-03T18:59:57,108 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T18:59:57,118 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32769-0x1019c8fba300001, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:59:57,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T18:59:57,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T18:59:57,120 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T18:59:57,122 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T18:59:57,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=db5a5ccf5be8,32769,1733252334945}] 2024-12-03T18:59:57,304 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T18:59:57,307 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32905, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T18:59:57,317 INFO [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T18:59:57,317 INFO [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T18:59:57,318 INFO [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T18:59:57,321 INFO [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=db5a5ccf5be8%2C32769%2C1733252334945.meta, suffix=.meta, logDir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/WALs/db5a5ccf5be8,32769,1733252334945, archiveDir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/oldWALs, maxLogs=32 2024-12-03T18:59:57,339 DEBUG [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/WALs/db5a5ccf5be8,32769,1733252334945/db5a5ccf5be8%2C32769%2C1733252334945.meta.1733252397323.meta, exclude list is [], retry=0 2024-12-03T18:59:57,343 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35847,DS-66553b08-2ade-4967-b00a-55fa72770989,DISK] 2024-12-03T18:59:57,347 INFO [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/WALs/db5a5ccf5be8,32769,1733252334945/db5a5ccf5be8%2C32769%2C1733252334945.meta.1733252397323.meta 2024-12-03T18:59:57,347 DEBUG [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36947:36947)] 2024-12-03T18:59:57,348 DEBUG [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T18:59:57,350 DEBUG [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T18:59:57,353 DEBUG [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T18:59:57,360 INFO [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T18:59:57,365 DEBUG [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T18:59:57,366 DEBUG [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:59:57,366 DEBUG [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T18:59:57,367 DEBUG [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T18:59:57,370 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T18:59:57,372 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T18:59:57,372 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:59:57,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T18:59:57,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T18:59:57,375 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T18:59:57,375 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:59:57,376 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T18:59:57,376 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T18:59:57,378 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T18:59:57,378 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:59:57,379 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T18:59:57,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T18:59:57,382 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T18:59:57,382 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:59:57,383 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T18:59:57,383 DEBUG [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T18:59:57,385 DEBUG [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740 2024-12-03T18:59:57,388 DEBUG [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740 2024-12-03T18:59:57,390 DEBUG [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T18:59:57,391 DEBUG [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T18:59:57,392 DEBUG [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T18:59:57,395 DEBUG [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T18:59:57,396 INFO [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71996513, jitterRate=0.0728316456079483}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T18:59:57,397 DEBUG [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T18:59:57,399 DEBUG [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733252397367Writing region info on filesystem at 1733252397367Initializing all the Stores at 1733252397369 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733252397369Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733252397370 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733252397370Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733252397370Cleaning up temporary data from old regions at 1733252397391 (+21 ms)Running coprocessor post-open hooks at 1733252397397 (+6 ms)Region opened successfully at 1733252397398 (+1 ms) 2024-12-03T18:59:57,407 INFO [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733252397296 2024-12-03T18:59:57,419 DEBUG [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T18:59:57,420 INFO [RS_OPEN_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T18:59:57,421 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T18:59:57,424 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as db5a5ccf5be8,32769,1733252334945, state=OPEN 2024-12-03T18:59:57,460 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32769-0x1019c8fba300001, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T18:59:57,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T18:59:57,460 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T18:59:57,460 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T18:59:57,460 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=db5a5ccf5be8,32769,1733252334945 2024-12-03T18:59:57,466 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T18:59:57,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=db5a5ccf5be8,32769,1733252334945 in 336 msec 2024-12-03T18:59:57,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T18:59:57,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 59.9060 sec 2024-12-03T18:59:57,477 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T18:59:57,477 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T18:59:57,495 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T18:59:57,497 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db5a5ccf5be8,32769,1733252334945, seqNum=-1] 2024-12-03T18:59:57,516 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T18:59:57,518 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45005, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T18:59:57,541 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1 mins, 0.966 sec 2024-12-03T18:59:57,541 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733252397541, completionTime=-1 2024-12-03T18:59:57,544 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-03T18:59:57,544 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T18:59:57,570 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-03T18:59:57,570 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733252457570 2024-12-03T18:59:57,570 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733252517570 2024-12-03T18:59:57,570 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-12-03T18:59:57,572 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db5a5ccf5be8,33603,1733252334072-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T18:59:57,572 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db5a5ccf5be8,33603,1733252334072-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T18:59:57,573 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db5a5ccf5be8,33603,1733252334072-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T18:59:57,574 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-db5a5ccf5be8:33603, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T18:59:57,575 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T18:59:57,575 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T18:59:57,581 DEBUG [master/db5a5ccf5be8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T18:59:57,604 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 62.470sec 2024-12-03T18:59:57,605 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T18:59:57,607 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T18:59:57,608 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T18:59:57,608 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T18:59:57,608 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T18:59:57,609 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db5a5ccf5be8,33603,1733252334072-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T18:59:57,610 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db5a5ccf5be8,33603,1733252334072-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T18:59:57,618 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T18:59:57,620 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T18:59:57,620 INFO [master/db5a5ccf5be8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=db5a5ccf5be8,33603,1733252334072-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T18:59:57,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f875e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T18:59:57,714 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T18:59:57,714 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T18:59:57,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db5a5ccf5be8,33603,-1 for getting cluster id 2024-12-03T18:59:57,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T18:59:57,729 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65bbdcb6-df76-46bf-9410-d71c06dd4d2e' 2024-12-03T18:59:57,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T18:59:57,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65bbdcb6-df76-46bf-9410-d71c06dd4d2e" 2024-12-03T18:59:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b5905a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T18:59:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db5a5ccf5be8,33603,-1] 2024-12-03T18:59:57,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T18:59:57,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T18:59:57,741 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T18:59:57,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7ec3ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T18:59:57,744 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T18:59:57,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db5a5ccf5be8,32769,1733252334945, seqNum=-1] 2024-12-03T18:59:57,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T18:59:57,754 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T18:59:57,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=db5a5ccf5be8,33603,1733252334072 2024-12-03T18:59:57,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fb7c601, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T18:59:57,777 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request db5a5ccf5be8,33603,-1 for getting cluster id 2024-12-03T18:59:57,777 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T18:59:57,779 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65bbdcb6-df76-46bf-9410-d71c06dd4d2e' 2024-12-03T18:59:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T18:59:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65bbdcb6-df76-46bf-9410-d71c06dd4d2e" 2024-12-03T18:59:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68a1632, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T18:59:57,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [db5a5ccf5be8,33603,-1] 2024-12-03T18:59:57,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T18:59:57,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T18:59:57,783 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T18:59:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@293b62f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T18:59:57,819 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=208, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=247, ProcessCount=11, AvailableMemoryMB=6263 2024-12-03T18:59:57,826 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T18:59:57,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is db5a5ccf5be8,33603,1733252334072 2024-12-03T18:59:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1b0f8ba7 2024-12-03T18:59:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T18:59:57,842 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T18:59:57,848 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T18:59:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-12-03T18:59:57,861 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T18:59:57,864 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-12-03T18:59:57,864 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:59:57,866 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T18:59:57,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T18:59:57,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741835_1011 (size=428) 2024-12-03T18:59:58,281 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c6323caf0e6363fcfe4476d724157348, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e 2024-12-03T18:59:58,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741836_1012 (size=79) 2024-12-03T18:59:58,693 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:59:58,693 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing c6323caf0e6363fcfe4476d724157348, disabling compactions & flushes 2024-12-03T18:59:58,693 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348. 2024-12-03T18:59:58,693 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348. 2024-12-03T18:59:58,693 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348. after waiting 0 ms 2024-12-03T18:59:58,694 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348. 2024-12-03T18:59:58,694 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348. 2024-12-03T18:59:58,694 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for c6323caf0e6363fcfe4476d724157348: Waiting for close lock at 1733252398693Disabling compacts and flushes for region at 1733252398693Disabling writes for close at 1733252398693Writing region close event to WAL at 1733252398694 (+1 ms)Closed at 1733252398694 2024-12-03T18:59:58,697 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T18:59:58,704 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733252398697"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733252398697"}]},"ts":"1733252398697"} 2024-12-03T18:59:58,710 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T18:59:58,712 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T18:59:58,714 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733252398712"}]},"ts":"1733252398712"} 2024-12-03T18:59:58,719 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-12-03T18:59:58,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=c6323caf0e6363fcfe4476d724157348, ASSIGN}] 2024-12-03T18:59:58,723 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=c6323caf0e6363fcfe4476d724157348, ASSIGN 2024-12-03T18:59:58,725 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=c6323caf0e6363fcfe4476d724157348, ASSIGN; state=OFFLINE, location=db5a5ccf5be8,32769,1733252334945; forceNewPlan=false, retain=false 2024-12-03T18:59:58,877 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c6323caf0e6363fcfe4476d724157348, regionState=OPENING, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T18:59:58,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T18:59:58,883 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=c6323caf0e6363fcfe4476d724157348, ASSIGN because future has completed 2024-12-03T18:59:58,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6323caf0e6363fcfe4476d724157348, server=db5a5ccf5be8,32769,1733252334945}] 2024-12-03T18:59:59,047 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348. 2024-12-03T18:59:59,047 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c6323caf0e6363fcfe4476d724157348, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348.', STARTKEY => '', ENDKEY => ''} 2024-12-03T18:59:59,048 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync c6323caf0e6363fcfe4476d724157348 2024-12-03T18:59:59,048 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T18:59:59,048 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c6323caf0e6363fcfe4476d724157348 2024-12-03T18:59:59,048 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c6323caf0e6363fcfe4476d724157348 2024-12-03T18:59:59,051 INFO [StoreOpener-c6323caf0e6363fcfe4476d724157348-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c6323caf0e6363fcfe4476d724157348 2024-12-03T18:59:59,053 INFO [StoreOpener-c6323caf0e6363fcfe4476d724157348-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6323caf0e6363fcfe4476d724157348 columnFamilyName testFamily 2024-12-03T18:59:59,053 DEBUG [StoreOpener-c6323caf0e6363fcfe4476d724157348-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T18:59:59,055 INFO [StoreOpener-c6323caf0e6363fcfe4476d724157348-1 {}] regionserver.HStore(327): Store=c6323caf0e6363fcfe4476d724157348/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T18:59:59,055 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c6323caf0e6363fcfe4476d724157348 2024-12-03T18:59:59,057 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/c6323caf0e6363fcfe4476d724157348 2024-12-03T18:59:59,058 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/c6323caf0e6363fcfe4476d724157348 2024-12-03T18:59:59,059 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c6323caf0e6363fcfe4476d724157348 2024-12-03T18:59:59,059 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c6323caf0e6363fcfe4476d724157348 2024-12-03T18:59:59,063 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c6323caf0e6363fcfe4476d724157348 2024-12-03T18:59:59,067 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/c6323caf0e6363fcfe4476d724157348/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T18:59:59,068 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c6323caf0e6363fcfe4476d724157348; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68884530, jitterRate=0.026459485292434692}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T18:59:59,068 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c6323caf0e6363fcfe4476d724157348 2024-12-03T18:59:59,069 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c6323caf0e6363fcfe4476d724157348: Running coprocessor pre-open hook at 1733252399049Writing region info on filesystem at 1733252399049Initializing all the Stores at 1733252399050 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733252399050Cleaning up temporary data from old regions at 1733252399059 (+9 ms)Running coprocessor post-open hooks at 1733252399068 (+9 ms)Region opened successfully at 1733252399069 (+1 ms) 2024-12-03T18:59:59,071 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348., pid=6, masterSystemTime=1733252399041 2024-12-03T18:59:59,075 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348. 2024-12-03T18:59:59,075 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348. 2024-12-03T18:59:59,076 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c6323caf0e6363fcfe4476d724157348, regionState=OPEN, openSeqNum=2, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T18:59:59,080 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c6323caf0e6363fcfe4476d724157348, server=db5a5ccf5be8,32769,1733252334945 because future has completed 2024-12-03T18:59:59,087 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T18:59:59,087 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c6323caf0e6363fcfe4476d724157348, server=db5a5ccf5be8,32769,1733252334945 in 197 msec 2024-12-03T18:59:59,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T18:59:59,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=c6323caf0e6363fcfe4476d724157348, ASSIGN in 366 msec 2024-12-03T18:59:59,094 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T18:59:59,094 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733252399094"}]},"ts":"1733252399094"} 2024-12-03T18:59:59,097 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-12-03T18:59:59,099 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T18:59:59,103 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2460 sec 2024-12-03T19:00:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T19:00:00,890 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-12-03T19:00:00,890 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-12-03T19:00:00,892 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T19:00:00,898 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-12-03T19:00:00,899 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T19:00:00,899 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-12-03T19:00:00,902 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T19:00:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=db5a5ccf5be8,32769,1733252334945, seqNum=-1] 2024-12-03T19:00:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T19:00:00,907 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T19:00:00,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2] 2024-12-03T19:00:00,930 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T19:00:00,935 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-12-03T19:00:00,941 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-03T19:00:00,946 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-03T19:00:00,949 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-03T19:00:00,950 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=3311193839871459336 2024-12-03T19:00:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:45874 deadline: 1733252403948, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T19:00:00,959 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-03T19:00:00,961 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-03T19:00:00,962 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T19:00:00,964 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T19:00:00,965 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=3311193839871459338 2024-12-03T19:00:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:45874 deadline: 1733252403963, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T19:00:00,969 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T19:00:00,970 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=3311193839871459339 2024-12-03T19:00:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:45874 deadline: 1733252403969, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T19:00:00,974 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T19:00:00,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=3311193839871459340 2024-12-03T19:00:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:45874 deadline: 1733252403974, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T19:00:00,979 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T19:00:00,990 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=215 (was 208) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1842828572_22 at /127.0.0.1:49792 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=243 (was 247), ProcessCount=11 (was 11), AvailableMemoryMB=6235 (was 6263) 2024-12-03T19:00:00,999 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=215, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=243, ProcessCount=11, AvailableMemoryMB=6233 2024-12-03T19:00:01,003 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33603 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T19:00:01,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33603 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-12-03T19:00:01,008 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T19:00:01,008 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T19:00:01,008 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33603 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-12-03T19:00:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T19:00:01,011 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T19:00:01,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741837_1013 (size=414) 2024-12-03T19:00:01,424 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d73667e828a0d4ccf3242e1c18ec0530, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e 2024-12-03T19:00:01,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741838_1014 (size=65) 2024-12-03T19:00:01,432 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T19:00:01,432 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing d73667e828a0d4ccf3242e1c18ec0530, disabling compactions & flushes 2024-12-03T19:00:01,432 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530. 2024-12-03T19:00:01,432 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530. 2024-12-03T19:00:01,432 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530. after waiting 0 ms 2024-12-03T19:00:01,432 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530. 2024-12-03T19:00:01,432 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530. 2024-12-03T19:00:01,432 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for d73667e828a0d4ccf3242e1c18ec0530: Waiting for close lock at 1733252401432Disabling compacts and flushes for region at 1733252401432Disabling writes for close at 1733252401432Writing region close event to WAL at 1733252401432Closed at 1733252401432 2024-12-03T19:00:01,434 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T19:00:01,435 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1733252401434"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733252401434"}]},"ts":"1733252401434"} 2024-12-03T19:00:01,437 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T19:00:01,440 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T19:00:01,440 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733252401440"}]},"ts":"1733252401440"} 2024-12-03T19:00:01,444 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-03T19:00:01,444 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=d73667e828a0d4ccf3242e1c18ec0530, ASSIGN}] 2024-12-03T19:00:01,447 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=d73667e828a0d4ccf3242e1c18ec0530, ASSIGN 2024-12-03T19:00:01,449 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=d73667e828a0d4ccf3242e1c18ec0530, ASSIGN; state=OFFLINE, location=db5a5ccf5be8,32769,1733252334945; forceNewPlan=false, retain=false 2024-12-03T19:00:01,599 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=d73667e828a0d4ccf3242e1c18ec0530, regionState=OPENING, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T19:00:01,603 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=d73667e828a0d4ccf3242e1c18ec0530, ASSIGN because future has completed 2024-12-03T19:00:01,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure d73667e828a0d4ccf3242e1c18ec0530, server=db5a5ccf5be8,32769,1733252334945}] 2024-12-03T19:00:01,764 INFO [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530. 2024-12-03T19:00:01,764 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => d73667e828a0d4ccf3242e1c18ec0530, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530.', STARTKEY => '', ENDKEY => ''} 2024-12-03T19:00:01,765 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner d73667e828a0d4ccf3242e1c18ec0530 2024-12-03T19:00:01,765 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T19:00:01,765 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for d73667e828a0d4ccf3242e1c18ec0530 2024-12-03T19:00:01,765 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for d73667e828a0d4ccf3242e1c18ec0530 2024-12-03T19:00:01,768 INFO [StoreOpener-d73667e828a0d4ccf3242e1c18ec0530-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d73667e828a0d4ccf3242e1c18ec0530 2024-12-03T19:00:01,770 INFO [StoreOpener-d73667e828a0d4ccf3242e1c18ec0530-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d73667e828a0d4ccf3242e1c18ec0530 columnFamilyName testFamily 2024-12-03T19:00:01,770 DEBUG [StoreOpener-d73667e828a0d4ccf3242e1c18ec0530-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T19:00:01,771 INFO [StoreOpener-d73667e828a0d4ccf3242e1c18ec0530-1 {}] regionserver.HStore(327): Store=d73667e828a0d4ccf3242e1c18ec0530/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T19:00:01,771 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for d73667e828a0d4ccf3242e1c18ec0530 2024-12-03T19:00:01,772 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnOpenScanner/d73667e828a0d4ccf3242e1c18ec0530 2024-12-03T19:00:01,773 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnOpenScanner/d73667e828a0d4ccf3242e1c18ec0530 2024-12-03T19:00:01,774 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for d73667e828a0d4ccf3242e1c18ec0530 2024-12-03T19:00:01,774 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for d73667e828a0d4ccf3242e1c18ec0530 2024-12-03T19:00:01,776 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for d73667e828a0d4ccf3242e1c18ec0530 2024-12-03T19:00:01,780 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnOpenScanner/d73667e828a0d4ccf3242e1c18ec0530/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T19:00:01,781 INFO [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened d73667e828a0d4ccf3242e1c18ec0530; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66609252, jitterRate=-0.007444798946380615}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T19:00:01,782 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d73667e828a0d4ccf3242e1c18ec0530 2024-12-03T19:00:01,783 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for d73667e828a0d4ccf3242e1c18ec0530: Running coprocessor pre-open hook at 1733252401765Writing region info on filesystem at 1733252401765Initializing all the Stores at 1733252401767 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733252401767Cleaning up temporary data from old regions at 1733252401774 (+7 ms)Running coprocessor post-open hooks at 1733252401782 (+8 ms)Region opened successfully at 1733252401783 (+1 ms) 2024-12-03T19:00:01,785 INFO [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530., pid=9, masterSystemTime=1733252401758 2024-12-03T19:00:01,789 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530. 2024-12-03T19:00:01,789 INFO [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530. 2024-12-03T19:00:01,790 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=d73667e828a0d4ccf3242e1c18ec0530, regionState=OPEN, openSeqNum=2, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T19:00:01,794 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure d73667e828a0d4ccf3242e1c18ec0530, server=db5a5ccf5be8,32769,1733252334945 because future has completed 2024-12-03T19:00:01,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-03T19:00:01,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure d73667e828a0d4ccf3242e1c18ec0530, server=db5a5ccf5be8,32769,1733252334945 in 192 msec 2024-12-03T19:00:01,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T19:00:01,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=d73667e828a0d4ccf3242e1c18ec0530, ASSIGN in 356 msec 2024-12-03T19:00:01,808 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T19:00:01,808 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733252401808"}]},"ts":"1733252401808"} 2024-12-03T19:00:01,812 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-03T19:00:01,814 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T19:00:01,818 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 811 msec 2024-12-03T19:00:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T19:00:02,029 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-12-03T19:00:02,029 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-03T19:00:02,030 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T19:00:02,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-03T19:00:02,035 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T19:00:02,035 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-12-03T19:00:02,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2] 2024-12-03T19:00:02,054 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T19:00:02,054 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-03T19:00:02,059 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T19:00:02,061 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32769 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-03T19:00:02,903 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T19:00:02,930 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-12-03T19:00:02,932 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T19:00:02,933 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-12-03T19:00:04,470 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T19:00:04,470 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T19:00:04,471 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T19:00:04,471 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T19:00:04,471 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-03T19:00:04,471 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T19:00:04,472 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-03T19:00:04,472 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-03T19:00:08,076 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=db5a5ccf5be8:32769 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6013ms, rpcTimeout=5999ms 2024-12-03T19:00:08,077 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6013ms, rpcTimeout=5999ms 2024-12-03T19:00:08,077 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2 from cache 2024-12-03T19:00:08,085 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=218 (was 215) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=232 (was 243), ProcessCount=11 (was 11), AvailableMemoryMB=6214 (was 6233) 2024-12-03T19:00:08,093 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=218, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=232, ProcessCount=11, AvailableMemoryMB=6214 2024-12-03T19:00:08,095 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33603 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T19:00:08,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33603 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-12-03T19:00:08,099 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T19:00:08,099 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T19:00:08,099 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33603 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-12-03T19:00:08,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-03T19:00:08,101 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T19:00:08,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741839_1015 (size=412) 2024-12-03T19:00:08,518 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c67d00014c6ed04bf717813407dc89f8, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e 2024-12-03T19:00:08,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741840_1016 (size=63) 2024-12-03T19:00:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-12-03T19:00:08,934 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T19:00:08,934 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing c67d00014c6ed04bf717813407dc89f8, disabling compactions & flushes 2024-12-03T19:00:08,934 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8. 2024-12-03T19:00:08,934 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8. 2024-12-03T19:00:08,934 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8. after waiting 0 ms 2024-12-03T19:00:08,934 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8. 2024-12-03T19:00:08,934 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8. 2024-12-03T19:00:08,934 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for c67d00014c6ed04bf717813407dc89f8: Waiting for close lock at 1733252408934Disabling compacts and flushes for region at 1733252408934Disabling writes for close at 1733252408934Writing region close event to WAL at 1733252408934Closed at 1733252408934 2024-12-03T19:00:08,937 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T19:00:08,937 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1733252408937"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733252408937"}]},"ts":"1733252408937"} 2024-12-03T19:00:08,941 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T19:00:08,943 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T19:00:08,944 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733252408943"}]},"ts":"1733252408943"} 2024-12-03T19:00:08,947 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-03T19:00:08,948 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=c67d00014c6ed04bf717813407dc89f8, ASSIGN}] 2024-12-03T19:00:08,950 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=c67d00014c6ed04bf717813407dc89f8, ASSIGN 2024-12-03T19:00:08,952 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=c67d00014c6ed04bf717813407dc89f8, ASSIGN; state=OFFLINE, location=db5a5ccf5be8,32769,1733252334945; forceNewPlan=false, retain=false 2024-12-03T19:00:09,103 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=c67d00014c6ed04bf717813407dc89f8, regionState=OPENING, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T19:00:09,107 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=c67d00014c6ed04bf717813407dc89f8, ASSIGN because future has completed 2024-12-03T19:00:09,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure c67d00014c6ed04bf717813407dc89f8, server=db5a5ccf5be8,32769,1733252334945}] 2024-12-03T19:00:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-03T19:00:09,267 INFO [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8. 2024-12-03T19:00:09,267 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => c67d00014c6ed04bf717813407dc89f8, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8.', STARTKEY => '', ENDKEY => ''} 2024-12-03T19:00:09,267 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync c67d00014c6ed04bf717813407dc89f8 2024-12-03T19:00:09,268 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T19:00:09,268 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for c67d00014c6ed04bf717813407dc89f8 2024-12-03T19:00:09,268 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for c67d00014c6ed04bf717813407dc89f8 2024-12-03T19:00:09,270 INFO [StoreOpener-c67d00014c6ed04bf717813407dc89f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c67d00014c6ed04bf717813407dc89f8 2024-12-03T19:00:09,273 INFO [StoreOpener-c67d00014c6ed04bf717813407dc89f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c67d00014c6ed04bf717813407dc89f8 columnFamilyName testFamily 2024-12-03T19:00:09,273 DEBUG [StoreOpener-c67d00014c6ed04bf717813407dc89f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T19:00:09,274 INFO [StoreOpener-c67d00014c6ed04bf717813407dc89f8-1 {}] regionserver.HStore(327): Store=c67d00014c6ed04bf717813407dc89f8/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T19:00:09,274 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for c67d00014c6ed04bf717813407dc89f8 2024-12-03T19:00:09,275 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnNextAsync/c67d00014c6ed04bf717813407dc89f8 2024-12-03T19:00:09,276 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnNextAsync/c67d00014c6ed04bf717813407dc89f8 2024-12-03T19:00:09,276 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for c67d00014c6ed04bf717813407dc89f8 2024-12-03T19:00:09,277 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for c67d00014c6ed04bf717813407dc89f8 2024-12-03T19:00:09,279 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for c67d00014c6ed04bf717813407dc89f8 2024-12-03T19:00:09,283 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnNextAsync/c67d00014c6ed04bf717813407dc89f8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T19:00:09,284 INFO [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened c67d00014c6ed04bf717813407dc89f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69709592, jitterRate=0.03875386714935303}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T19:00:09,284 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c67d00014c6ed04bf717813407dc89f8 2024-12-03T19:00:09,284 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for c67d00014c6ed04bf717813407dc89f8: Running coprocessor pre-open hook at 1733252409268Writing region info on filesystem at 1733252409268Initializing all the Stores at 1733252409270 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733252409270Cleaning up temporary data from old regions at 1733252409277 (+7 ms)Running coprocessor post-open hooks at 1733252409284 (+7 ms)Region opened successfully at 1733252409284 2024-12-03T19:00:09,286 INFO [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8., pid=12, masterSystemTime=1733252409261 2024-12-03T19:00:09,289 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8. 2024-12-03T19:00:09,289 INFO [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8. 2024-12-03T19:00:09,290 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=c67d00014c6ed04bf717813407dc89f8, regionState=OPEN, openSeqNum=2, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T19:00:09,293 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure c67d00014c6ed04bf717813407dc89f8, server=db5a5ccf5be8,32769,1733252334945 because future has completed 2024-12-03T19:00:09,300 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-03T19:00:09,302 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure c67d00014c6ed04bf717813407dc89f8, server=db5a5ccf5be8,32769,1733252334945 in 189 msec 2024-12-03T19:00:09,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-03T19:00:09,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=c67d00014c6ed04bf717813407dc89f8, ASSIGN in 352 msec 2024-12-03T19:00:09,306 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T19:00:09,306 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733252409306"}]},"ts":"1733252409306"} 2024-12-03T19:00:09,309 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-03T19:00:09,310 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T19:00:09,313 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2160 sec 2024-12-03T19:00:11,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-03T19:00:11,119 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-12-03T19:00:11,119 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-03T19:00:11,120 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T19:00:11,124 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-03T19:00:11,125 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T19:00:11,125 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-12-03T19:00:11,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2] 2024-12-03T19:00:11,142 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T19:00:11,142 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-03T19:00:11,145 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T19:00:11,148 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T19:00:11,150 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T19:00:11,151 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32769 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-03T19:00:14,784 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T19:00:14,802 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-03T19:00:14,802 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T19:00:14,935 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-12-03T19:00:17,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32769 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '3311193839871459349' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T19:00:17,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32769 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:45874 deadline: 1733252423159, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '3311193839871459349' does not exist 2024-12-03T19:00:17,163 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to db5a5ccf5be8,32769,1733252334945 for closing scanner id = 3311193839871459349 for c67d00014c6ed04bf717813407dc89f8 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '3311193839871459349' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T19:00:17,166 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=220 (was 218) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=235 (was 232) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7116 (was 6214) - AvailableMemoryMB LEAK? - 2024-12-03T19:00:17,173 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=220, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=235, ProcessCount=11, AvailableMemoryMB=7115 2024-12-03T19:00:17,175 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T19:00:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-12-03T19:00:17,179 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T19:00:17,179 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T19:00:17,180 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-12-03T19:00:17,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T19:00:17,181 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T19:00:17,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741841_1017 (size=423) 2024-12-03T19:00:17,593 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e1f3687225c56a2a0c1813c8d16663d7, NAME => 'testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e 2024-12-03T19:00:17,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741842_1018 (size=74) 2024-12-03T19:00:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-12-03T19:00:18,004 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T19:00:18,004 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing e1f3687225c56a2a0c1813c8d16663d7, disabling compactions & flushes 2024-12-03T19:00:18,004 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7. 2024-12-03T19:00:18,004 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7. 2024-12-03T19:00:18,004 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7. after waiting 0 ms 2024-12-03T19:00:18,004 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7. 2024-12-03T19:00:18,004 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7. 2024-12-03T19:00:18,004 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for e1f3687225c56a2a0c1813c8d16663d7: Waiting for close lock at 1733252418004Disabling compacts and flushes for region at 1733252418004Disabling writes for close at 1733252418004Writing region close event to WAL at 1733252418004Closed at 1733252418004 2024-12-03T19:00:18,006 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T19:00:18,007 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733252418006"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733252418006"}]},"ts":"1733252418006"} 2024-12-03T19:00:18,010 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T19:00:18,011 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T19:00:18,011 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733252418011"}]},"ts":"1733252418011"} 2024-12-03T19:00:18,013 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-12-03T19:00:18,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=e1f3687225c56a2a0c1813c8d16663d7, ASSIGN}] 2024-12-03T19:00:18,015 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=e1f3687225c56a2a0c1813c8d16663d7, ASSIGN 2024-12-03T19:00:18,017 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=e1f3687225c56a2a0c1813c8d16663d7, ASSIGN; state=OFFLINE, location=db5a5ccf5be8,32769,1733252334945; forceNewPlan=false, retain=false 2024-12-03T19:00:18,168 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=e1f3687225c56a2a0c1813c8d16663d7, regionState=OPENING, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T19:00:18,175 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=e1f3687225c56a2a0c1813c8d16663d7, ASSIGN because future has completed 2024-12-03T19:00:18,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1f3687225c56a2a0c1813c8d16663d7, server=db5a5ccf5be8,32769,1733252334945}] 2024-12-03T19:00:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T19:00:18,335 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7. 2024-12-03T19:00:18,336 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => e1f3687225c56a2a0c1813c8d16663d7, NAME => 'testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7.', STARTKEY => '', ENDKEY => ''} 2024-12-03T19:00:18,336 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException e1f3687225c56a2a0c1813c8d16663d7 2024-12-03T19:00:18,336 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T19:00:18,337 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for e1f3687225c56a2a0c1813c8d16663d7 2024-12-03T19:00:18,337 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for e1f3687225c56a2a0c1813c8d16663d7 2024-12-03T19:00:18,340 INFO [StoreOpener-e1f3687225c56a2a0c1813c8d16663d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e1f3687225c56a2a0c1813c8d16663d7 2024-12-03T19:00:18,344 INFO [StoreOpener-e1f3687225c56a2a0c1813c8d16663d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1f3687225c56a2a0c1813c8d16663d7 columnFamilyName testFamily 2024-12-03T19:00:18,344 DEBUG [StoreOpener-e1f3687225c56a2a0c1813c8d16663d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T19:00:18,345 INFO [StoreOpener-e1f3687225c56a2a0c1813c8d16663d7-1 {}] regionserver.HStore(327): Store=e1f3687225c56a2a0c1813c8d16663d7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T19:00:18,345 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for e1f3687225c56a2a0c1813c8d16663d7 2024-12-03T19:00:18,347 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testRetryOutOfOrderScannerNextException/e1f3687225c56a2a0c1813c8d16663d7 2024-12-03T19:00:18,347 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testRetryOutOfOrderScannerNextException/e1f3687225c56a2a0c1813c8d16663d7 2024-12-03T19:00:18,348 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for e1f3687225c56a2a0c1813c8d16663d7 2024-12-03T19:00:18,348 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for e1f3687225c56a2a0c1813c8d16663d7 2024-12-03T19:00:18,351 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for e1f3687225c56a2a0c1813c8d16663d7 2024-12-03T19:00:18,355 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testRetryOutOfOrderScannerNextException/e1f3687225c56a2a0c1813c8d16663d7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T19:00:18,356 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened e1f3687225c56a2a0c1813c8d16663d7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67591249, jitterRate=0.007188096642494202}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T19:00:18,356 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e1f3687225c56a2a0c1813c8d16663d7 2024-12-03T19:00:18,356 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for e1f3687225c56a2a0c1813c8d16663d7: Running coprocessor pre-open hook at 1733252418337Writing region info on filesystem at 1733252418337Initializing all the Stores at 1733252418339 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733252418340 (+1 ms)Cleaning up temporary data from old regions at 1733252418348 (+8 ms)Running coprocessor post-open hooks at 1733252418356 (+8 ms)Region opened successfully at 1733252418356 2024-12-03T19:00:18,358 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7., pid=15, masterSystemTime=1733252418329 2024-12-03T19:00:18,360 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7. 2024-12-03T19:00:18,360 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7. 2024-12-03T19:00:18,361 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=e1f3687225c56a2a0c1813c8d16663d7, regionState=OPEN, openSeqNum=2, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T19:00:18,364 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1f3687225c56a2a0c1813c8d16663d7, server=db5a5ccf5be8,32769,1733252334945 because future has completed 2024-12-03T19:00:18,368 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-03T19:00:18,369 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure e1f3687225c56a2a0c1813c8d16663d7, server=db5a5ccf5be8,32769,1733252334945 in 189 msec 2024-12-03T19:00:18,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T19:00:18,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=e1f3687225c56a2a0c1813c8d16663d7, ASSIGN in 355 msec 2024-12-03T19:00:18,372 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T19:00:18,373 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733252418372"}]},"ts":"1733252418372"} 2024-12-03T19:00:18,375 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-12-03T19:00:18,376 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T19:00:18,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2010 sec 2024-12-03T19:00:20,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T19:00:20,209 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-12-03T19:00:20,209 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-12-03T19:00:20,210 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T19:00:20,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-12-03T19:00:20,214 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T19:00:20,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-12-03T19:00:20,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2] 2024-12-03T19:00:20,230 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T19:00:20,230 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-12-03T19:00:20,235 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-03T19:00:20,239 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-03T19:00:20,241 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-03T19:00:20,242 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=3311193839871459353 2024-12-03T19:00:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:45874 deadline: 1733252423240, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T19:00:20,246 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-03T19:00:20,248 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-03T19:00:20,248 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T19:00:20,250 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T19:00:20,252 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=3311193839871459355 2024-12-03T19:00:20,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:45874 deadline: 1733252423250, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T19:00:20,256 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T19:00:20,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=3311193839871459356 2024-12-03T19:00:20,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:45874 deadline: 1733252423255, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T19:00:20,260 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T19:00:20,261 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=3311193839871459357 2024-12-03T19:00:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:45874 deadline: 1733252423260, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T19:00:20,265 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T19:00:20,275 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 220) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1842828572_22 at /127.0.0.1:45592 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=233 (was 235), ProcessCount=11 (was 11), AvailableMemoryMB=6980 (was 7115) 2024-12-03T19:00:20,285 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=233, ProcessCount=11, AvailableMemoryMB=6980 2024-12-03T19:00:20,289 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T19:00:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-12-03T19:00:20,293 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T19:00:20,293 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T19:00:20,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-12-03T19:00:20,295 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T19:00:20,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-03T19:00:20,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741843_1019 (size=418) 2024-12-03T19:00:20,307 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d8016fe6cfb7745a592bb2fbc748686f, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e 2024-12-03T19:00:20,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741844_1020 (size=69) 2024-12-03T19:00:20,716 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T19:00:20,716 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing d8016fe6cfb7745a592bb2fbc748686f, disabling compactions & flushes 2024-12-03T19:00:20,716 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f. 2024-12-03T19:00:20,716 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f. 2024-12-03T19:00:20,716 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f. after waiting 0 ms 2024-12-03T19:00:20,716 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f. 2024-12-03T19:00:20,716 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f. 2024-12-03T19:00:20,716 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for d8016fe6cfb7745a592bb2fbc748686f: Waiting for close lock at 1733252420716Disabling compacts and flushes for region at 1733252420716Disabling writes for close at 1733252420716Writing region close event to WAL at 1733252420716Closed at 1733252420716 2024-12-03T19:00:20,718 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T19:00:20,718 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1733252420718"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733252420718"}]},"ts":"1733252420718"} 2024-12-03T19:00:20,721 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T19:00:20,722 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T19:00:20,722 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733252420722"}]},"ts":"1733252420722"} 2024-12-03T19:00:20,724 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-03T19:00:20,725 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d8016fe6cfb7745a592bb2fbc748686f, ASSIGN}] 2024-12-03T19:00:20,726 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d8016fe6cfb7745a592bb2fbc748686f, ASSIGN 2024-12-03T19:00:20,727 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d8016fe6cfb7745a592bb2fbc748686f, ASSIGN; state=OFFLINE, location=db5a5ccf5be8,32769,1733252334945; forceNewPlan=false, retain=false 2024-12-03T19:00:20,878 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=d8016fe6cfb7745a592bb2fbc748686f, regionState=OPENING, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T19:00:20,883 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d8016fe6cfb7745a592bb2fbc748686f, ASSIGN because future has completed 2024-12-03T19:00:20,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8016fe6cfb7745a592bb2fbc748686f, server=db5a5ccf5be8,32769,1733252334945}] 2024-12-03T19:00:20,936 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-12-03T19:00:21,048 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f. 2024-12-03T19:00:21,049 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => d8016fe6cfb7745a592bb2fbc748686f, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f.', STARTKEY => '', ENDKEY => ''} 2024-12-03T19:00:21,049 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner d8016fe6cfb7745a592bb2fbc748686f 2024-12-03T19:00:21,049 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T19:00:21,049 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for d8016fe6cfb7745a592bb2fbc748686f 2024-12-03T19:00:21,049 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for d8016fe6cfb7745a592bb2fbc748686f 2024-12-03T19:00:21,052 INFO [StoreOpener-d8016fe6cfb7745a592bb2fbc748686f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d8016fe6cfb7745a592bb2fbc748686f 2024-12-03T19:00:21,054 INFO [StoreOpener-d8016fe6cfb7745a592bb2fbc748686f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d8016fe6cfb7745a592bb2fbc748686f columnFamilyName testFamily 2024-12-03T19:00:21,055 DEBUG [StoreOpener-d8016fe6cfb7745a592bb2fbc748686f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T19:00:21,055 INFO [StoreOpener-d8016fe6cfb7745a592bb2fbc748686f-1 {}] regionserver.HStore(327): Store=d8016fe6cfb7745a592bb2fbc748686f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T19:00:21,056 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for d8016fe6cfb7745a592bb2fbc748686f 2024-12-03T19:00:21,057 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnOpenScanner/d8016fe6cfb7745a592bb2fbc748686f 2024-12-03T19:00:21,058 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnOpenScanner/d8016fe6cfb7745a592bb2fbc748686f 2024-12-03T19:00:21,058 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for d8016fe6cfb7745a592bb2fbc748686f 2024-12-03T19:00:21,058 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for d8016fe6cfb7745a592bb2fbc748686f 2024-12-03T19:00:21,061 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for d8016fe6cfb7745a592bb2fbc748686f 2024-12-03T19:00:21,064 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnOpenScanner/d8016fe6cfb7745a592bb2fbc748686f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T19:00:21,064 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened d8016fe6cfb7745a592bb2fbc748686f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62028703, jitterRate=-0.0757002979516983}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T19:00:21,065 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d8016fe6cfb7745a592bb2fbc748686f 2024-12-03T19:00:21,065 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for d8016fe6cfb7745a592bb2fbc748686f: Running coprocessor pre-open hook at 1733252421050Writing region info on filesystem at 1733252421050Initializing all the Stores at 1733252421052 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733252421052Cleaning up temporary data from old regions at 1733252421058 (+6 ms)Running coprocessor post-open hooks at 1733252421065 (+7 ms)Region opened successfully at 1733252421065 2024-12-03T19:00:21,066 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f., pid=18, masterSystemTime=1733252421038 2024-12-03T19:00:21,069 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f. 2024-12-03T19:00:21,069 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f. 2024-12-03T19:00:21,070 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=d8016fe6cfb7745a592bb2fbc748686f, regionState=OPEN, openSeqNum=2, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T19:00:21,072 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure d8016fe6cfb7745a592bb2fbc748686f, server=db5a5ccf5be8,32769,1733252334945 because future has completed 2024-12-03T19:00:21,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-03T19:00:21,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure d8016fe6cfb7745a592bb2fbc748686f, server=db5a5ccf5be8,32769,1733252334945 in 191 msec 2024-12-03T19:00:21,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-03T19:00:21,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d8016fe6cfb7745a592bb2fbc748686f, ASSIGN in 352 msec 2024-12-03T19:00:21,081 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T19:00:21,081 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733252421081"}]},"ts":"1733252421081"} 2024-12-03T19:00:21,084 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-03T19:00:21,085 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T19:00:21,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 796 msec 2024-12-03T19:00:21,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-03T19:00:21,310 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-12-03T19:00:21,310 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-03T19:00:21,311 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T19:00:21,316 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-03T19:00:21,317 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T19:00:21,317 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-12-03T19:00:21,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2] 2024-12-03T19:00:21,331 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T19:00:21,331 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-03T19:00:21,334 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T19:00:21,336 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T19:00:22,337 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=db5a5ccf5be8:32769 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-12-03T19:00:22,337 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-12-03T19:00:22,338 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2 from cache 2024-12-03T19:00:22,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-12-03T19:00:23,121 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T19:00:23,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2] 2024-12-03T19:00:23,351 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T19:00:23,839 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T19:00:24,337 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=db5a5ccf5be8:32769 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=987ms, rpcTimeout=984ms 2024-12-03T19:00:24,337 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=987ms, rpcTimeout=984ms 2024-12-03T19:00:24,337 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2 from cache 2024-12-03T19:00:24,352 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223 (was 223), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=214 (was 233), ProcessCount=11 (was 11), AvailableMemoryMB=6964 (was 6980) 2024-12-03T19:00:24,363 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=214, ProcessCount=11, AvailableMemoryMB=6963 2024-12-03T19:00:24,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T19:00:24,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-12-03T19:00:24,398 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T19:00:24,398 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T19:00:24,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-12-03T19:00:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T19:00:24,401 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T19:00:24,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741845_1021 (size=416) 2024-12-03T19:00:24,470 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-03T19:00:24,470 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-12-03T19:00:24,471 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-03T19:00:24,471 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-03T19:00:24,818 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0de1d919473dc75d50db231a4234192c, NAME => 'testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e 2024-12-03T19:00:24,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741846_1022 (size=67) 2024-12-03T19:00:24,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-12-03T19:00:25,231 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T19:00:25,231 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 0de1d919473dc75d50db231a4234192c, disabling compactions & flushes 2024-12-03T19:00:25,231 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c. 2024-12-03T19:00:25,231 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c. 2024-12-03T19:00:25,232 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c. after waiting 0 ms 2024-12-03T19:00:25,232 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c. 2024-12-03T19:00:25,232 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c. 2024-12-03T19:00:25,233 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0de1d919473dc75d50db231a4234192c: Waiting for close lock at 1733252425231Disabling compacts and flushes for region at 1733252425231Disabling writes for close at 1733252425232 (+1 ms)Writing region close event to WAL at 1733252425232Closed at 1733252425232 2024-12-03T19:00:25,237 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T19:00:25,238 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1733252425237"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733252425237"}]},"ts":"1733252425237"} 2024-12-03T19:00:25,242 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T19:00:25,243 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T19:00:25,244 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733252425243"}]},"ts":"1733252425243"} 2024-12-03T19:00:25,246 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-03T19:00:25,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=0de1d919473dc75d50db231a4234192c, ASSIGN}] 2024-12-03T19:00:25,248 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=0de1d919473dc75d50db231a4234192c, ASSIGN 2024-12-03T19:00:25,249 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=0de1d919473dc75d50db231a4234192c, ASSIGN; state=OFFLINE, location=db5a5ccf5be8,32769,1733252334945; forceNewPlan=false, retain=false 2024-12-03T19:00:25,400 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=0de1d919473dc75d50db231a4234192c, regionState=OPENING, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T19:00:25,406 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=0de1d919473dc75d50db231a4234192c, ASSIGN because future has completed 2024-12-03T19:00:25,407 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0de1d919473dc75d50db231a4234192c, server=db5a5ccf5be8,32769,1733252334945}] 2024-12-03T19:00:25,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T19:00:25,573 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c. 2024-12-03T19:00:25,573 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 0de1d919473dc75d50db231a4234192c, NAME => 'testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c.', STARTKEY => '', ENDKEY => ''} 2024-12-03T19:00:25,574 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 0de1d919473dc75d50db231a4234192c 2024-12-03T19:00:25,574 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T19:00:25,574 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 0de1d919473dc75d50db231a4234192c 2024-12-03T19:00:25,574 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 0de1d919473dc75d50db231a4234192c 2024-12-03T19:00:25,577 INFO [StoreOpener-0de1d919473dc75d50db231a4234192c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0de1d919473dc75d50db231a4234192c 2024-12-03T19:00:25,579 INFO [StoreOpener-0de1d919473dc75d50db231a4234192c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0de1d919473dc75d50db231a4234192c columnFamilyName testFamily 2024-12-03T19:00:25,579 DEBUG [StoreOpener-0de1d919473dc75d50db231a4234192c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T19:00:25,580 INFO [StoreOpener-0de1d919473dc75d50db231a4234192c-1 {}] regionserver.HStore(327): Store=0de1d919473dc75d50db231a4234192c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T19:00:25,580 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 0de1d919473dc75d50db231a4234192c 2024-12-03T19:00:25,582 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnNextAsync/0de1d919473dc75d50db231a4234192c 2024-12-03T19:00:25,582 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnNextAsync/0de1d919473dc75d50db231a4234192c 2024-12-03T19:00:25,583 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 0de1d919473dc75d50db231a4234192c 2024-12-03T19:00:25,583 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 0de1d919473dc75d50db231a4234192c 2024-12-03T19:00:25,586 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 0de1d919473dc75d50db231a4234192c 2024-12-03T19:00:25,590 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnNextAsync/0de1d919473dc75d50db231a4234192c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T19:00:25,591 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 0de1d919473dc75d50db231a4234192c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66813241, jitterRate=-0.004405125975608826}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T19:00:25,592 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0de1d919473dc75d50db231a4234192c 2024-12-03T19:00:25,592 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 0de1d919473dc75d50db231a4234192c: Running coprocessor pre-open hook at 1733252425574Writing region info on filesystem at 1733252425574Initializing all the Stores at 1733252425576 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733252425576Cleaning up temporary data from old regions at 1733252425583 (+7 ms)Running coprocessor post-open hooks at 1733252425592 (+9 ms)Region opened successfully at 1733252425592 2024-12-03T19:00:25,594 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c., pid=21, masterSystemTime=1733252425561 2024-12-03T19:00:25,596 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c. 2024-12-03T19:00:25,597 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c. 2024-12-03T19:00:25,597 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=0de1d919473dc75d50db231a4234192c, regionState=OPEN, openSeqNum=2, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T19:00:25,600 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0de1d919473dc75d50db231a4234192c, server=db5a5ccf5be8,32769,1733252334945 because future has completed 2024-12-03T19:00:25,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-03T19:00:25,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 0de1d919473dc75d50db231a4234192c, server=db5a5ccf5be8,32769,1733252334945 in 195 msec 2024-12-03T19:00:25,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-03T19:00:25,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=0de1d919473dc75d50db231a4234192c, ASSIGN in 357 msec 2024-12-03T19:00:25,608 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T19:00:25,608 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733252425608"}]},"ts":"1733252425608"} 2024-12-03T19:00:25,610 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-03T19:00:25,611 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T19:00:25,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2450 sec 2024-12-03T19:00:26,938 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-12-03T19:00:26,938 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-12-03T19:00:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T19:00:27,420 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-12-03T19:00:27,420 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-03T19:00:27,420 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T19:00:27,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-03T19:00:27,429 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T19:00:27,430 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-12-03T19:00:27,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2] 2024-12-03T19:00:27,443 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T19:00:27,443 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-03T19:00:27,446 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T19:00:27,449 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T19:00:27,450 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T19:00:27,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-03T19:00:30,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32769 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '3311193839871459368' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T19:00:30,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32769 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:45874 deadline: 1733252431464, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '3311193839871459368' does not exist 2024-12-03T19:00:30,466 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to db5a5ccf5be8,32769,1733252334945 for closing scanner id = 3311193839871459368 for 0de1d919473dc75d50db231a4234192c of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '3311193839871459368' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T19:00:30,472 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 223), OpenFileDescriptor=448 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=197 (was 214), ProcessCount=11 (was 11), AvailableMemoryMB=6959 (was 6963) 2024-12-03T19:00:30,480 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=197, ProcessCount=11, AvailableMemoryMB=6958 2024-12-03T19:00:30,482 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T19:00:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-12-03T19:00:30,485 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T19:00:30,486 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T19:00:30,486 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-12-03T19:00:30,487 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T19:00:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-03T19:00:30,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741847_1023 (size=411) 2024-12-03T19:00:30,903 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 903a9fc1a5dbd7bf3f74d3054afe53d5, NAME => 'testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e 2024-12-03T19:00:30,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741848_1024 (size=62) 2024-12-03T19:00:30,915 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T19:00:30,915 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 903a9fc1a5dbd7bf3f74d3054afe53d5, disabling compactions & flushes 2024-12-03T19:00:30,916 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5. 2024-12-03T19:00:30,916 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5. 2024-12-03T19:00:30,916 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5. after waiting 0 ms 2024-12-03T19:00:30,916 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5. 2024-12-03T19:00:30,916 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5. 2024-12-03T19:00:30,916 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 903a9fc1a5dbd7bf3f74d3054afe53d5: Waiting for close lock at 1733252430915Disabling compacts and flushes for region at 1733252430915Disabling writes for close at 1733252430916 (+1 ms)Writing region close event to WAL at 1733252430916Closed at 1733252430916 2024-12-03T19:00:30,918 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T19:00:30,918 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1733252430918"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733252430918"}]},"ts":"1733252430918"} 2024-12-03T19:00:30,921 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T19:00:30,922 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T19:00:30,922 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733252430922"}]},"ts":"1733252430922"} 2024-12-03T19:00:30,925 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-03T19:00:30,926 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=903a9fc1a5dbd7bf3f74d3054afe53d5, ASSIGN}] 2024-12-03T19:00:30,927 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=903a9fc1a5dbd7bf3f74d3054afe53d5, ASSIGN 2024-12-03T19:00:30,928 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=903a9fc1a5dbd7bf3f74d3054afe53d5, ASSIGN; state=OFFLINE, location=db5a5ccf5be8,32769,1733252334945; forceNewPlan=false, retain=false 2024-12-03T19:00:30,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-12-03T19:00:31,077 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T19:00:31,080 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=903a9fc1a5dbd7bf3f74d3054afe53d5, regionState=OPENING, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T19:00:31,086 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=903a9fc1a5dbd7bf3f74d3054afe53d5, ASSIGN because future has completed 2024-12-03T19:00:31,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 903a9fc1a5dbd7bf3f74d3054afe53d5, server=db5a5ccf5be8,32769,1733252334945}] 2024-12-03T19:00:31,253 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5. 2024-12-03T19:00:31,253 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 903a9fc1a5dbd7bf3f74d3054afe53d5, NAME => 'testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5.', STARTKEY => '', ENDKEY => ''} 2024-12-03T19:00:31,253 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 903a9fc1a5dbd7bf3f74d3054afe53d5 2024-12-03T19:00:31,253 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T19:00:31,254 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 903a9fc1a5dbd7bf3f74d3054afe53d5 2024-12-03T19:00:31,254 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 903a9fc1a5dbd7bf3f74d3054afe53d5 2024-12-03T19:00:31,256 INFO [StoreOpener-903a9fc1a5dbd7bf3f74d3054afe53d5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 903a9fc1a5dbd7bf3f74d3054afe53d5 2024-12-03T19:00:31,259 INFO [StoreOpener-903a9fc1a5dbd7bf3f74d3054afe53d5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 903a9fc1a5dbd7bf3f74d3054afe53d5 columnFamilyName testFamily 2024-12-03T19:00:31,259 DEBUG [StoreOpener-903a9fc1a5dbd7bf3f74d3054afe53d5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T19:00:31,260 INFO [StoreOpener-903a9fc1a5dbd7bf3f74d3054afe53d5-1 {}] regionserver.HStore(327): Store=903a9fc1a5dbd7bf3f74d3054afe53d5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T19:00:31,260 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 903a9fc1a5dbd7bf3f74d3054afe53d5 2024-12-03T19:00:31,263 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnNext/903a9fc1a5dbd7bf3f74d3054afe53d5 2024-12-03T19:00:31,264 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnNext/903a9fc1a5dbd7bf3f74d3054afe53d5 2024-12-03T19:00:31,264 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 903a9fc1a5dbd7bf3f74d3054afe53d5 2024-12-03T19:00:31,265 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 903a9fc1a5dbd7bf3f74d3054afe53d5 2024-12-03T19:00:31,268 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 903a9fc1a5dbd7bf3f74d3054afe53d5 2024-12-03T19:00:31,271 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnNext/903a9fc1a5dbd7bf3f74d3054afe53d5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T19:00:31,272 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 903a9fc1a5dbd7bf3f74d3054afe53d5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66310688, jitterRate=-0.011893749237060547}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T19:00:31,272 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 903a9fc1a5dbd7bf3f74d3054afe53d5 2024-12-03T19:00:31,273 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 903a9fc1a5dbd7bf3f74d3054afe53d5: Running coprocessor pre-open hook at 1733252431254Writing region info on filesystem at 1733252431254Initializing all the Stores at 1733252431256 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733252431256Cleaning up temporary data from old regions at 1733252431265 (+9 ms)Running coprocessor post-open hooks at 1733252431272 (+7 ms)Region opened successfully at 1733252431273 (+1 ms) 2024-12-03T19:00:31,274 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5., pid=24, masterSystemTime=1733252431240 2024-12-03T19:00:31,277 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5. 2024-12-03T19:00:31,277 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5. 2024-12-03T19:00:31,278 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=903a9fc1a5dbd7bf3f74d3054afe53d5, regionState=OPEN, openSeqNum=2, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T19:00:31,280 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 903a9fc1a5dbd7bf3f74d3054afe53d5, server=db5a5ccf5be8,32769,1733252334945 because future has completed 2024-12-03T19:00:31,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-03T19:00:31,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 903a9fc1a5dbd7bf3f74d3054afe53d5, server=db5a5ccf5be8,32769,1733252334945 in 195 msec 2024-12-03T19:00:31,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-03T19:00:31,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=903a9fc1a5dbd7bf3f74d3054afe53d5, ASSIGN in 359 msec 2024-12-03T19:00:31,289 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T19:00:31,290 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733252431289"}]},"ts":"1733252431289"} 2024-12-03T19:00:31,293 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-03T19:00:31,294 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T19:00:31,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 812 msec 2024-12-03T19:00:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-03T19:00:31,501 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-12-03T19:00:31,501 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-03T19:00:31,501 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T19:00:31,512 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-03T19:00:31,512 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T19:00:31,512 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-12-03T19:00:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2] 2024-12-03T19:00:31,524 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T19:00:31,524 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-03T19:00:31,528 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T19:00:31,530 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T19:00:31,531 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T19:00:31,532 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-03T19:00:32,940 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-12-03T19:00:34,470 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-03T19:00:34,470 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-03T19:00:34,471 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-03T19:00:34,471 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T19:00:34,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32769 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '3311193839871459372' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T19:00:34,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32769 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:45874 deadline: 1733252435542, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '3311193839871459372' does not exist 2024-12-03T19:00:34,546 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to db5a5ccf5be8,32769,1733252334945 for closing scanner id = 3311193839871459372 for 903a9fc1a5dbd7bf3f74d3054afe53d5 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '3311193839871459372' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T19:00:34,552 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223 (was 222) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1842828572_22 at /127.0.0.1:35524 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=181 (was 197), ProcessCount=11 (was 11), AvailableMemoryMB=6953 (was 6958) 2024-12-03T19:00:34,561 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=181, ProcessCount=11, AvailableMemoryMB=6953 2024-12-03T19:00:34,563 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33603 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T19:00:34,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33603 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-12-03T19:00:34,566 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T19:00:34,567 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T19:00:34,567 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33603 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-12-03T19:00:34,568 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T19:00:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T19:00:34,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741849_1025 (size=419) 2024-12-03T19:00:34,983 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 11506a0fb3ece4c0913a479f1e8f8d28, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e 2024-12-03T19:00:34,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741850_1026 (size=70) 2024-12-03T19:00:35,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-12-03T19:00:35,396 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T19:00:35,396 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 11506a0fb3ece4c0913a479f1e8f8d28, disabling compactions & flushes 2024-12-03T19:00:35,396 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28. 2024-12-03T19:00:35,396 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28. 2024-12-03T19:00:35,396 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28. after waiting 0 ms 2024-12-03T19:00:35,397 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28. 2024-12-03T19:00:35,397 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28. 2024-12-03T19:00:35,397 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 11506a0fb3ece4c0913a479f1e8f8d28: Waiting for close lock at 1733252435396Disabling compacts and flushes for region at 1733252435396Disabling writes for close at 1733252435397 (+1 ms)Writing region close event to WAL at 1733252435397Closed at 1733252435397 2024-12-03T19:00:35,401 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T19:00:35,401 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1733252435401"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733252435401"}]},"ts":"1733252435401"} 2024-12-03T19:00:35,405 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T19:00:35,406 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T19:00:35,406 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733252435406"}]},"ts":"1733252435406"} 2024-12-03T19:00:35,409 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-03T19:00:35,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=11506a0fb3ece4c0913a479f1e8f8d28, ASSIGN}] 2024-12-03T19:00:35,411 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=11506a0fb3ece4c0913a479f1e8f8d28, ASSIGN 2024-12-03T19:00:35,412 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=11506a0fb3ece4c0913a479f1e8f8d28, ASSIGN; state=OFFLINE, location=db5a5ccf5be8,32769,1733252334945; forceNewPlan=false, retain=false 2024-12-03T19:00:35,563 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=11506a0fb3ece4c0913a479f1e8f8d28, regionState=OPENING, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T19:00:35,568 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=11506a0fb3ece4c0913a479f1e8f8d28, ASSIGN because future has completed 2024-12-03T19:00:35,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11506a0fb3ece4c0913a479f1e8f8d28, server=db5a5ccf5be8,32769,1733252334945}] 2024-12-03T19:00:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T19:00:35,789 INFO [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28. 2024-12-03T19:00:35,790 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 11506a0fb3ece4c0913a479f1e8f8d28, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28.', STARTKEY => '', ENDKEY => ''} 2024-12-03T19:00:35,790 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 11506a0fb3ece4c0913a479f1e8f8d28 2024-12-03T19:00:35,791 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T19:00:35,791 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 11506a0fb3ece4c0913a479f1e8f8d28 2024-12-03T19:00:35,791 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 11506a0fb3ece4c0913a479f1e8f8d28 2024-12-03T19:00:35,793 INFO [StoreOpener-11506a0fb3ece4c0913a479f1e8f8d28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 11506a0fb3ece4c0913a479f1e8f8d28 2024-12-03T19:00:35,796 INFO [StoreOpener-11506a0fb3ece4c0913a479f1e8f8d28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 11506a0fb3ece4c0913a479f1e8f8d28 columnFamilyName testFamily 2024-12-03T19:00:35,796 DEBUG [StoreOpener-11506a0fb3ece4c0913a479f1e8f8d28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T19:00:35,796 INFO [StoreOpener-11506a0fb3ece4c0913a479f1e8f8d28-1 {}] regionserver.HStore(327): Store=11506a0fb3ece4c0913a479f1e8f8d28/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T19:00:35,796 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 11506a0fb3ece4c0913a479f1e8f8d28 2024-12-03T19:00:35,797 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/11506a0fb3ece4c0913a479f1e8f8d28 2024-12-03T19:00:35,798 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/11506a0fb3ece4c0913a479f1e8f8d28 2024-12-03T19:00:35,798 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 11506a0fb3ece4c0913a479f1e8f8d28 2024-12-03T19:00:35,798 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 11506a0fb3ece4c0913a479f1e8f8d28 2024-12-03T19:00:35,800 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 11506a0fb3ece4c0913a479f1e8f8d28 2024-12-03T19:00:35,803 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/11506a0fb3ece4c0913a479f1e8f8d28/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T19:00:35,804 INFO [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 11506a0fb3ece4c0913a479f1e8f8d28; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65792894, jitterRate=-0.0196094810962677}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T19:00:35,804 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 11506a0fb3ece4c0913a479f1e8f8d28 2024-12-03T19:00:35,805 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 11506a0fb3ece4c0913a479f1e8f8d28: Running coprocessor pre-open hook at 1733252435791Writing region info on filesystem at 1733252435791Initializing all the Stores at 1733252435793 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733252435793Cleaning up temporary data from old regions at 1733252435798 (+5 ms)Running coprocessor post-open hooks at 1733252435804 (+6 ms)Region opened successfully at 1733252435805 (+1 ms) 2024-12-03T19:00:35,806 INFO [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28., pid=27, masterSystemTime=1733252435779 2024-12-03T19:00:35,808 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28. 2024-12-03T19:00:35,809 INFO [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28. 2024-12-03T19:00:35,809 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=11506a0fb3ece4c0913a479f1e8f8d28, regionState=OPEN, openSeqNum=2, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T19:00:35,812 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 11506a0fb3ece4c0913a479f1e8f8d28, server=db5a5ccf5be8,32769,1733252334945 because future has completed 2024-12-03T19:00:35,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-03T19:00:35,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 11506a0fb3ece4c0913a479f1e8f8d28, server=db5a5ccf5be8,32769,1733252334945 in 244 msec 2024-12-03T19:00:35,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-03T19:00:35,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=11506a0fb3ece4c0913a479f1e8f8d28, ASSIGN in 407 msec 2024-12-03T19:00:35,820 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T19:00:35,820 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733252435820"}]},"ts":"1733252435820"} 2024-12-03T19:00:35,823 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-03T19:00:35,824 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T19:00:35,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2620 sec 2024-12-03T19:00:36,758 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T19:00:37,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T19:00:37,600 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-12-03T19:00:37,600 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-03T19:00:37,600 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T19:00:37,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-03T19:00:37,609 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T19:00:37,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-12-03T19:00:37,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2] 2024-12-03T19:00:37,621 INFO [master/db5a5ccf5be8:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-03T19:00:37,621 INFO [master/db5a5ccf5be8:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-03T19:00:37,623 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T19:00:37,623 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-03T19:00:37,626 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T19:00:37,628 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32769 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-03T19:00:38,943 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-12-03T19:00:43,637 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=db5a5ccf5be8:32769 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6010ms, rpcTimeout=5999ms 2024-12-03T19:00:43,638 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6010ms, rpcTimeout=5999ms 2024-12-03T19:00:43,638 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2 from cache 2024-12-03T19:00:43,653 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 223), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=166 (was 181), ProcessCount=11 (was 11), AvailableMemoryMB=6942 (was 6953) 2024-12-03T19:00:43,663 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=166, ProcessCount=11, AvailableMemoryMB=6942 2024-12-03T19:00:43,666 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33603 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T19:00:43,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33603 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-12-03T19:00:43,669 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T19:00:43,669 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T19:00:43,669 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33603 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-12-03T19:00:43,671 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T19:00:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T19:00:43,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741851_1027 (size=407) 2024-12-03T19:00:44,086 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e2eb87f7aa3da3d8944542aebd8f95a1, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e 2024-12-03T19:00:44,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741852_1028 (size=58) 2024-12-03T19:00:44,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-12-03T19:00:44,470 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-03T19:00:44,470 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T19:00:44,500 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T19:00:44,501 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing e2eb87f7aa3da3d8944542aebd8f95a1, disabling compactions & flushes 2024-12-03T19:00:44,501 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1. 2024-12-03T19:00:44,501 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1. 2024-12-03T19:00:44,501 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1. after waiting 0 ms 2024-12-03T19:00:44,501 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1. 2024-12-03T19:00:44,501 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1. 2024-12-03T19:00:44,501 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for e2eb87f7aa3da3d8944542aebd8f95a1: Waiting for close lock at 1733252444500Disabling compacts and flushes for region at 1733252444500Disabling writes for close at 1733252444501 (+1 ms)Writing region close event to WAL at 1733252444501Closed at 1733252444501 2024-12-03T19:00:44,505 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T19:00:44,505 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1733252444505"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733252444505"}]},"ts":"1733252444505"} 2024-12-03T19:00:44,509 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T19:00:44,511 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T19:00:44,511 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733252444511"}]},"ts":"1733252444511"} 2024-12-03T19:00:44,515 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-03T19:00:44,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=e2eb87f7aa3da3d8944542aebd8f95a1, ASSIGN}] 2024-12-03T19:00:44,517 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=e2eb87f7aa3da3d8944542aebd8f95a1, ASSIGN 2024-12-03T19:00:44,518 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=e2eb87f7aa3da3d8944542aebd8f95a1, ASSIGN; state=OFFLINE, location=db5a5ccf5be8,32769,1733252334945; forceNewPlan=false, retain=false 2024-12-03T19:00:44,669 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=e2eb87f7aa3da3d8944542aebd8f95a1, regionState=OPENING, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T19:00:44,675 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=e2eb87f7aa3da3d8944542aebd8f95a1, ASSIGN because future has completed 2024-12-03T19:00:44,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2eb87f7aa3da3d8944542aebd8f95a1, server=db5a5ccf5be8,32769,1733252334945}] 2024-12-03T19:00:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T19:00:44,839 INFO [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1. 2024-12-03T19:00:44,839 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => e2eb87f7aa3da3d8944542aebd8f95a1, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1.', STARTKEY => '', ENDKEY => ''} 2024-12-03T19:00:44,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:00:44,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T19:00:44,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:00:44,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:00:44,843 INFO [StoreOpener-e2eb87f7aa3da3d8944542aebd8f95a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:00:44,846 INFO [StoreOpener-e2eb87f7aa3da3d8944542aebd8f95a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2eb87f7aa3da3d8944542aebd8f95a1 columnFamilyName testFamily 2024-12-03T19:00:44,846 DEBUG [StoreOpener-e2eb87f7aa3da3d8944542aebd8f95a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T19:00:44,847 INFO [StoreOpener-e2eb87f7aa3da3d8944542aebd8f95a1-1 {}] regionserver.HStore(327): Store=e2eb87f7aa3da3d8944542aebd8f95a1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T19:00:44,847 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:00:44,849 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnNext/e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:00:44,850 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnNext/e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:00:44,851 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:00:44,851 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:00:44,854 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:00:44,857 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnNext/e2eb87f7aa3da3d8944542aebd8f95a1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T19:00:44,858 INFO [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened e2eb87f7aa3da3d8944542aebd8f95a1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73252816, jitterRate=0.0915520191192627}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T19:00:44,858 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:00:44,859 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for e2eb87f7aa3da3d8944542aebd8f95a1: Running coprocessor pre-open hook at 1733252444840Writing region info on filesystem at 1733252444840Initializing all the Stores at 1733252444842 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733252444842Cleaning up temporary data from old regions at 1733252444851 (+9 ms)Running coprocessor post-open hooks at 1733252444858 (+7 ms)Region opened successfully at 1733252444859 (+1 ms) 2024-12-03T19:00:44,860 INFO [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1., pid=30, masterSystemTime=1733252444832 2024-12-03T19:00:44,862 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1. 2024-12-03T19:00:44,862 INFO [RS_OPEN_PRIORITY_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1. 2024-12-03T19:00:44,863 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=e2eb87f7aa3da3d8944542aebd8f95a1, regionState=OPEN, openSeqNum=2, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T19:00:44,865 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure e2eb87f7aa3da3d8944542aebd8f95a1, server=db5a5ccf5be8,32769,1733252334945 because future has completed 2024-12-03T19:00:44,869 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-03T19:00:44,869 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure e2eb87f7aa3da3d8944542aebd8f95a1, server=db5a5ccf5be8,32769,1733252334945 in 190 msec 2024-12-03T19:00:44,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-03T19:00:44,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=e2eb87f7aa3da3d8944542aebd8f95a1, ASSIGN in 354 msec 2024-12-03T19:00:44,872 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T19:00:44,872 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733252444872"}]},"ts":"1733252444872"} 2024-12-03T19:00:44,874 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-03T19:00:44,875 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T19:00:44,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2090 sec 2024-12-03T19:00:44,949 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-12-03T19:00:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T19:00:46,721 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-12-03T19:00:46,721 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-03T19:00:46,722 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T19:00:46,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-03T19:00:46,735 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T19:00:46,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-12-03T19:00:46,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2] 2024-12-03T19:00:46,749 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T19:00:46,749 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-03T19:00:46,751 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T19:00:46,753 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T19:00:46,754 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T19:00:46,755 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32769 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-03T19:00:50,364 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T19:00:52,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32769 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '3311193839871459380' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T19:00:52,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32769 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:45874 deadline: 1733252458758, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '3311193839871459380' does not exist 2024-12-03T19:00:52,763 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to db5a5ccf5be8,32769,1733252334945 for closing scanner id = 3311193839871459380 for e2eb87f7aa3da3d8944542aebd8f95a1 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '3311193839871459380' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T19:00:52,771 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=157 (was 166), ProcessCount=11 (was 11), AvailableMemoryMB=6927 (was 6942) 2024-12-03T19:00:52,778 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=157, ProcessCount=11, AvailableMemoryMB=6927 2024-12-03T19:00:52,780 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T19:00:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-12-03T19:00:52,783 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T19:00:52,783 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T19:00:52,783 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-12-03T19:00:52,784 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T19:00:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T19:00:52,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741853_1029 (size=423) 2024-12-03T19:00:53,121 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T19:00:53,198 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9b50dcf94758d2169683cf2c5fa74390, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e 2024-12-03T19:00:53,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741854_1030 (size=74) 2024-12-03T19:00:53,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-12-03T19:00:53,615 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T19:00:53,616 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 9b50dcf94758d2169683cf2c5fa74390, disabling compactions & flushes 2024-12-03T19:00:53,616 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390. 2024-12-03T19:00:53,616 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390. 2024-12-03T19:00:53,616 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390. after waiting 0 ms 2024-12-03T19:00:53,616 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390. 2024-12-03T19:00:53,616 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390. 2024-12-03T19:00:53,617 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9b50dcf94758d2169683cf2c5fa74390: Waiting for close lock at 1733252453615Disabling compacts and flushes for region at 1733252453615Disabling writes for close at 1733252453616 (+1 ms)Writing region close event to WAL at 1733252453616Closed at 1733252453616 2024-12-03T19:00:53,621 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T19:00:53,622 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733252453622"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733252453622"}]},"ts":"1733252453622"} 2024-12-03T19:00:53,626 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T19:00:53,627 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T19:00:53,627 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733252453627"}]},"ts":"1733252453627"} 2024-12-03T19:00:53,629 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-03T19:00:53,630 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9b50dcf94758d2169683cf2c5fa74390, ASSIGN}] 2024-12-03T19:00:53,631 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9b50dcf94758d2169683cf2c5fa74390, ASSIGN 2024-12-03T19:00:53,632 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9b50dcf94758d2169683cf2c5fa74390, ASSIGN; state=OFFLINE, location=db5a5ccf5be8,32769,1733252334945; forceNewPlan=false, retain=false 2024-12-03T19:00:53,783 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=9b50dcf94758d2169683cf2c5fa74390, regionState=OPENING, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T19:00:53,789 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9b50dcf94758d2169683cf2c5fa74390, ASSIGN because future has completed 2024-12-03T19:00:53,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T19:00:53,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b50dcf94758d2169683cf2c5fa74390, server=db5a5ccf5be8,32769,1733252334945}] 2024-12-03T19:00:53,955 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390. 2024-12-03T19:00:53,956 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 9b50dcf94758d2169683cf2c5fa74390, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390.', STARTKEY => '', ENDKEY => ''} 2024-12-03T19:00:53,956 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 9b50dcf94758d2169683cf2c5fa74390 2024-12-03T19:00:53,956 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T19:00:53,956 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 9b50dcf94758d2169683cf2c5fa74390 2024-12-03T19:00:53,957 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 9b50dcf94758d2169683cf2c5fa74390 2024-12-03T19:00:53,960 INFO [StoreOpener-9b50dcf94758d2169683cf2c5fa74390-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9b50dcf94758d2169683cf2c5fa74390 2024-12-03T19:00:53,963 INFO [StoreOpener-9b50dcf94758d2169683cf2c5fa74390-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b50dcf94758d2169683cf2c5fa74390 columnFamilyName testFamily 2024-12-03T19:00:53,963 DEBUG [StoreOpener-9b50dcf94758d2169683cf2c5fa74390-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T19:00:53,964 INFO [StoreOpener-9b50dcf94758d2169683cf2c5fa74390-1 {}] regionserver.HStore(327): Store=9b50dcf94758d2169683cf2c5fa74390/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T19:00:53,964 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 9b50dcf94758d2169683cf2c5fa74390 2024-12-03T19:00:53,966 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnOpenScannerAsync/9b50dcf94758d2169683cf2c5fa74390 2024-12-03T19:00:53,967 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnOpenScannerAsync/9b50dcf94758d2169683cf2c5fa74390 2024-12-03T19:00:53,968 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 9b50dcf94758d2169683cf2c5fa74390 2024-12-03T19:00:53,968 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 9b50dcf94758d2169683cf2c5fa74390 2024-12-03T19:00:53,970 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 9b50dcf94758d2169683cf2c5fa74390 2024-12-03T19:00:53,973 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnOpenScannerAsync/9b50dcf94758d2169683cf2c5fa74390/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T19:00:53,973 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 9b50dcf94758d2169683cf2c5fa74390; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65357381, jitterRate=-0.026099130511283875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T19:00:53,974 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9b50dcf94758d2169683cf2c5fa74390 2024-12-03T19:00:53,974 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 9b50dcf94758d2169683cf2c5fa74390: Running coprocessor pre-open hook at 1733252453957Writing region info on filesystem at 1733252453957Initializing all the Stores at 1733252453959 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733252453959Cleaning up temporary data from old regions at 1733252453968 (+9 ms)Running coprocessor post-open hooks at 1733252453974 (+6 ms)Region opened successfully at 1733252453974 2024-12-03T19:00:53,975 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390., pid=33, masterSystemTime=1733252453944 2024-12-03T19:00:53,977 DEBUG [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390. 2024-12-03T19:00:53,978 INFO [RS_OPEN_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390. 2024-12-03T19:00:53,978 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=9b50dcf94758d2169683cf2c5fa74390, regionState=OPEN, openSeqNum=2, regionLocation=db5a5ccf5be8,32769,1733252334945 2024-12-03T19:00:53,981 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b50dcf94758d2169683cf2c5fa74390, server=db5a5ccf5be8,32769,1733252334945 because future has completed 2024-12-03T19:00:53,984 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-03T19:00:53,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 9b50dcf94758d2169683cf2c5fa74390, server=db5a5ccf5be8,32769,1733252334945 in 192 msec 2024-12-03T19:00:53,987 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-03T19:00:53,987 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=9b50dcf94758d2169683cf2c5fa74390, ASSIGN in 355 msec 2024-12-03T19:00:53,988 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T19:00:53,988 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733252453988"}]},"ts":"1733252453988"} 2024-12-03T19:00:53,991 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-03T19:00:53,992 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T19:00:53,994 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.2120 sec 2024-12-03T19:00:54,470 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-12-03T19:00:54,470 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T19:00:54,471 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-03T19:00:54,471 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-03T19:00:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T19:00:55,810 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-12-03T19:00:55,810 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-03T19:00:55,811 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T19:00:55,819 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-03T19:00:55,820 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T19:00:55,820 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-12-03T19:00:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2] 2024-12-03T19:00:55,836 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T19:00:55,836 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-03T19:00:55,839 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T19:00:55,841 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T19:00:56,846 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=db5a5ccf5be8:32769 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-12-03T19:00:56,846 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-12-03T19:00:56,846 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2 from cache 2024-12-03T19:00:56,952 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-12-03T19:00:57,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-12-03T19:00:57,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2] 2024-12-03T19:00:57,865 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32769 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T19:00:58,847 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=db5a5ccf5be8:32769 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=982ms, rpcTimeout=975ms 2024-12-03T19:00:58,848 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=982ms, rpcTimeout=975ms 2024-12-03T19:00:58,848 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390., hostname=db5a5ccf5be8,32769,1733252334945, seqNum=2 from cache 2024-12-03T19:00:58,864 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=221 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=160 (was 157) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6921 (was 6927) 2024-12-03T19:00:58,864 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T19:00:58,865 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T19:00:58,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T19:00:58,871 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T19:00:58,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T19:00:58,871 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T19:00:58,872 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T19:00:58,872 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T19:00:58,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T19:00:58,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T19:00:58,872 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T19:00:58,872 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T19:00:58,872 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1822065101, stopped=false 2024-12-03T19:00:58,872 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=db5a5ccf5be8,33603,1733252334072 2024-12-03T19:00:58,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T19:00:58,951 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32769-0x1019c8fba300001, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T19:00:58,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T19:00:58,951 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32769-0x1019c8fba300001, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T19:00:58,951 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T19:00:58,954 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T19:00:58,954 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T19:00:58,954 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32769-0x1019c8fba300001, quorum=127.0.0.1:59911, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T19:00:58,954 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T19:00:58,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T19:00:58,955 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'db5a5ccf5be8,32769,1733252334945' ***** 2024-12-03T19:00:58,955 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T19:00:58,955 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T19:00:58,956 INFO [RS:0;db5a5ccf5be8:32769 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T19:00:58,956 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T19:00:58,956 INFO [RS:0;db5a5ccf5be8:32769 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T19:00:58,956 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(3091): Received CLOSE for e1f3687225c56a2a0c1813c8d16663d7 2024-12-03T19:00:58,957 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(3091): Received CLOSE for c6323caf0e6363fcfe4476d724157348 2024-12-03T19:00:58,957 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(3091): Received CLOSE for 9b50dcf94758d2169683cf2c5fa74390 2024-12-03T19:00:58,957 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(3091): Received CLOSE for d73667e828a0d4ccf3242e1c18ec0530 2024-12-03T19:00:58,957 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(3091): Received CLOSE for 0de1d919473dc75d50db231a4234192c 2024-12-03T19:00:58,957 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(3091): Received CLOSE for 11506a0fb3ece4c0913a479f1e8f8d28 2024-12-03T19:00:58,957 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e1f3687225c56a2a0c1813c8d16663d7, disabling compactions & flushes 2024-12-03T19:00:58,958 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7. 2024-12-03T19:00:58,958 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(3091): Received CLOSE for 903a9fc1a5dbd7bf3f74d3054afe53d5 2024-12-03T19:00:58,958 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7. 2024-12-03T19:00:58,958 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(3091): Received CLOSE for e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:00:58,958 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7. after waiting 0 ms 2024-12-03T19:00:58,958 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(3091): Received CLOSE for c67d00014c6ed04bf717813407dc89f8 2024-12-03T19:00:58,958 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7. 2024-12-03T19:00:58,958 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(3091): Received CLOSE for d8016fe6cfb7745a592bb2fbc748686f 2024-12-03T19:00:58,958 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(959): stopping server db5a5ccf5be8,32769,1733252334945 2024-12-03T19:00:58,958 INFO [RS:0;db5a5ccf5be8:32769 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T19:00:58,958 INFO [RS:0;db5a5ccf5be8:32769 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;db5a5ccf5be8:32769. 2024-12-03T19:00:58,958 DEBUG [RS:0;db5a5ccf5be8:32769 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T19:00:58,958 DEBUG [RS:0;db5a5ccf5be8:32769 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T19:00:58,959 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T19:00:58,959 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T19:00:58,959 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T19:00:58,959 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T19:00:58,959 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-12-03T19:00:58,959 DEBUG [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T19:00:58,959 INFO [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T19:00:58,959 DEBUG [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T19:00:58,960 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on 0de1d919473dc75d50db231a4234192c, 11506a0fb3ece4c0913a479f1e8f8d28, 1588230740, 903a9fc1a5dbd7bf3f74d3054afe53d5, 9b50dcf94758d2169683cf2c5fa74390, c6323caf0e6363fcfe4476d724157348, c67d00014c6ed04bf717813407dc89f8, d73667e828a0d4ccf3242e1c18ec0530, d8016fe6cfb7745a592bb2fbc748686f, e1f3687225c56a2a0c1813c8d16663d7, e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:00:58,960 DEBUG [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T19:00:58,960 DEBUG [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T19:00:58,962 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e1f3687225c56a2a0c1813c8d16663d7 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T19:00:58,962 INFO [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-12-03T19:00:59,011 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testRetryOutOfOrderScannerNextException/e1f3687225c56a2a0c1813c8d16663d7/.tmp/testFamily/2fc25210256c44b2a68f23633427f386 is 61, key is row-0/testFamily:testQualifier/1733252420218/Put/seqid=0 2024-12-03T19:00:59,016 DEBUG [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740/.tmp/info/ff99e3a92bf9480687f8bf0daddd274a is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348./info:regioninfo/1733252399076/Put/seqid=0 2024-12-03T19:00:59,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741856_1032 (size=20564) 2024-12-03T19:00:59,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741855_1031 (size=5049) 2024-12-03T19:00:59,051 DEBUG [master/db5a5ccf5be8:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-03T19:00:59,160 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on 0de1d919473dc75d50db231a4234192c, 11506a0fb3ece4c0913a479f1e8f8d28, 1588230740, 903a9fc1a5dbd7bf3f74d3054afe53d5, 9b50dcf94758d2169683cf2c5fa74390, c6323caf0e6363fcfe4476d724157348, c67d00014c6ed04bf717813407dc89f8, d73667e828a0d4ccf3242e1c18ec0530, d8016fe6cfb7745a592bb2fbc748686f, e1f3687225c56a2a0c1813c8d16663d7, e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:00:59,360 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on 0de1d919473dc75d50db231a4234192c, 11506a0fb3ece4c0913a479f1e8f8d28, 1588230740, 903a9fc1a5dbd7bf3f74d3054afe53d5, 9b50dcf94758d2169683cf2c5fa74390, c6323caf0e6363fcfe4476d724157348, c67d00014c6ed04bf717813407dc89f8, d73667e828a0d4ccf3242e1c18ec0530, d8016fe6cfb7745a592bb2fbc748686f, e1f3687225c56a2a0c1813c8d16663d7, e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:00:59,366 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-03T19:00:59,423 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testRetryOutOfOrderScannerNextException/e1f3687225c56a2a0c1813c8d16663d7/.tmp/testFamily/2fc25210256c44b2a68f23633427f386 2024-12-03T19:00:59,423 INFO [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740/.tmp/info/ff99e3a92bf9480687f8bf0daddd274a 2024-12-03T19:00:59,458 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T19:00:59,464 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testRetryOutOfOrderScannerNextException/e1f3687225c56a2a0c1813c8d16663d7/.tmp/testFamily/2fc25210256c44b2a68f23633427f386 as hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testRetryOutOfOrderScannerNextException/e1f3687225c56a2a0c1813c8d16663d7/testFamily/2fc25210256c44b2a68f23633427f386 2024-12-03T19:00:59,474 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testRetryOutOfOrderScannerNextException/e1f3687225c56a2a0c1813c8d16663d7/testFamily/2fc25210256c44b2a68f23633427f386, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T19:00:59,482 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e1f3687225c56a2a0c1813c8d16663d7 in 519ms, sequenceid=8, compaction requested=false 2024-12-03T19:00:59,484 DEBUG [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740/.tmp/ns/031ae8ef56e441e0ace15b1511dc89fd is 43, key is default/ns:d/1733252397523/Put/seqid=0 2024-12-03T19:00:59,488 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testRetryOutOfOrderScannerNextException/e1f3687225c56a2a0c1813c8d16663d7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T19:00:59,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741857_1033 (size=5153) 2024-12-03T19:00:59,490 INFO [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740/.tmp/ns/031ae8ef56e441e0ace15b1511dc89fd 2024-12-03T19:00:59,490 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7. 2024-12-03T19:00:59,490 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e1f3687225c56a2a0c1813c8d16663d7: Waiting for close lock at 1733252458957Running coprocessor pre-close hooks at 1733252458957Disabling compacts and flushes for region at 1733252458957Disabling writes for close at 1733252458958 (+1 ms)Obtaining lock to block concurrent updates at 1733252458962 (+4 ms)Preparing flush snapshotting stores in e1f3687225c56a2a0c1813c8d16663d7 at 1733252458962Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733252458968 (+6 ms)Flushing stores of testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7. at 1733252458970 (+2 ms)Flushing e1f3687225c56a2a0c1813c8d16663d7/testFamily: creating writer at 1733252458971 (+1 ms)Flushing e1f3687225c56a2a0c1813c8d16663d7/testFamily: appending metadata at 1733252459006 (+35 ms)Flushing e1f3687225c56a2a0c1813c8d16663d7/testFamily: closing flushed file at 1733252459009 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e07217f: reopening flushed file at 1733252459462 (+453 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e1f3687225c56a2a0c1813c8d16663d7 in 519ms, sequenceid=8, compaction requested=false at 1733252459482 (+20 ms)Writing region close event to WAL at 1733252459483 (+1 ms)Running coprocessor post-close hooks at 1733252459489 (+6 ms)Closed at 1733252459490 (+1 ms) 2024-12-03T19:00:59,491 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1733252417175.e1f3687225c56a2a0c1813c8d16663d7. 2024-12-03T19:00:59,491 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c6323caf0e6363fcfe4476d724157348, disabling compactions & flushes 2024-12-03T19:00:59,491 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348. 2024-12-03T19:00:59,491 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348. 2024-12-03T19:00:59,491 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348. after waiting 0 ms 2024-12-03T19:00:59,491 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348. 2024-12-03T19:00:59,492 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c6323caf0e6363fcfe4476d724157348 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T19:00:59,510 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/c6323caf0e6363fcfe4476d724157348/.tmp/testFamily/fea6c72a6a4948b1a9821de42b80d30b is 61, key is row-0/testFamily:testQualifier/1733252400915/Put/seqid=0 2024-12-03T19:00:59,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741858_1034 (size=5049) 2024-12-03T19:00:59,524 DEBUG [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740/.tmp/table/18519abbce5540d093daad0507bcd9ad is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1733252399094/Put/seqid=0 2024-12-03T19:00:59,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741859_1035 (size=6699) 2024-12-03T19:00:59,561 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on 0de1d919473dc75d50db231a4234192c, 11506a0fb3ece4c0913a479f1e8f8d28, 1588230740, 903a9fc1a5dbd7bf3f74d3054afe53d5, 9b50dcf94758d2169683cf2c5fa74390, c6323caf0e6363fcfe4476d724157348, c67d00014c6ed04bf717813407dc89f8, d73667e828a0d4ccf3242e1c18ec0530, d8016fe6cfb7745a592bb2fbc748686f, e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:00:59,761 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on 0de1d919473dc75d50db231a4234192c, 11506a0fb3ece4c0913a479f1e8f8d28, 1588230740, 903a9fc1a5dbd7bf3f74d3054afe53d5, 9b50dcf94758d2169683cf2c5fa74390, c6323caf0e6363fcfe4476d724157348, c67d00014c6ed04bf717813407dc89f8, d73667e828a0d4ccf3242e1c18ec0530, d8016fe6cfb7745a592bb2fbc748686f, e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:00:59,820 INFO [regionserver/db5a5ccf5be8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T19:00:59,821 INFO [regionserver/db5a5ccf5be8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T19:00:59,917 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/c6323caf0e6363fcfe4476d724157348/.tmp/testFamily/fea6c72a6a4948b1a9821de42b80d30b 2024-12-03T19:00:59,930 INFO [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740/.tmp/table/18519abbce5540d093daad0507bcd9ad 2024-12-03T19:00:59,931 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/c6323caf0e6363fcfe4476d724157348/.tmp/testFamily/fea6c72a6a4948b1a9821de42b80d30b as hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/c6323caf0e6363fcfe4476d724157348/testFamily/fea6c72a6a4948b1a9821de42b80d30b 2024-12-03T19:00:59,939 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/c6323caf0e6363fcfe4476d724157348/testFamily/fea6c72a6a4948b1a9821de42b80d30b, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T19:00:59,940 DEBUG [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740/.tmp/info/ff99e3a92bf9480687f8bf0daddd274a as hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740/info/ff99e3a92bf9480687f8bf0daddd274a 2024-12-03T19:00:59,940 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c6323caf0e6363fcfe4476d724157348 in 448ms, sequenceid=8, compaction requested=false 2024-12-03T19:00:59,945 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testRetryOutOfOrderScannerNextExceptionAsync/c6323caf0e6363fcfe4476d724157348/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T19:00:59,946 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348. 2024-12-03T19:00:59,946 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c6323caf0e6363fcfe4476d724157348: Waiting for close lock at 1733252459491Running coprocessor pre-close hooks at 1733252459491Disabling compacts and flushes for region at 1733252459491Disabling writes for close at 1733252459491Obtaining lock to block concurrent updates at 1733252459492 (+1 ms)Preparing flush snapshotting stores in c6323caf0e6363fcfe4476d724157348 at 1733252459492Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733252459492Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348. at 1733252459493 (+1 ms)Flushing c6323caf0e6363fcfe4476d724157348/testFamily: creating writer at 1733252459493Flushing c6323caf0e6363fcfe4476d724157348/testFamily: appending metadata at 1733252459510 (+17 ms)Flushing c6323caf0e6363fcfe4476d724157348/testFamily: closing flushed file at 1733252459510Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56e93cb4: reopening flushed file at 1733252459929 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c6323caf0e6363fcfe4476d724157348 in 448ms, sequenceid=8, compaction requested=false at 1733252459940 (+11 ms)Writing region close event to WAL at 1733252459941 (+1 ms)Running coprocessor post-close hooks at 1733252459946 (+5 ms)Closed at 1733252459946 2024-12-03T19:00:59,946 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733252397844.c6323caf0e6363fcfe4476d724157348. 2024-12-03T19:00:59,946 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9b50dcf94758d2169683cf2c5fa74390, disabling compactions & flushes 2024-12-03T19:00:59,946 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390. 2024-12-03T19:00:59,947 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390. 2024-12-03T19:00:59,947 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390. after waiting 0 ms 2024-12-03T19:00:59,947 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390. 2024-12-03T19:00:59,947 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9b50dcf94758d2169683cf2c5fa74390 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T19:00:59,948 INFO [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740/info/ff99e3a92bf9480687f8bf0daddd274a, entries=100, sequenceid=56, filesize=20.1 K 2024-12-03T19:00:59,950 DEBUG [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740/.tmp/ns/031ae8ef56e441e0ace15b1511dc89fd as hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740/ns/031ae8ef56e441e0ace15b1511dc89fd 2024-12-03T19:00:59,958 INFO [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740/ns/031ae8ef56e441e0ace15b1511dc89fd, entries=2, sequenceid=56, filesize=5.0 K 2024-12-03T19:00:59,959 DEBUG [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740/.tmp/table/18519abbce5540d093daad0507bcd9ad as hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740/table/18519abbce5540d093daad0507bcd9ad 2024-12-03T19:00:59,961 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-12-03T19:00:59,961 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1325): Online Regions={9b50dcf94758d2169683cf2c5fa74390=testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390., d73667e828a0d4ccf3242e1c18ec0530=hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530., 1588230740=hbase:meta,,1.1588230740, 0de1d919473dc75d50db231a4234192c=testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c., 11506a0fb3ece4c0913a479f1e8f8d28=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28., 903a9fc1a5dbd7bf3f74d3054afe53d5=testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5., e2eb87f7aa3da3d8944542aebd8f95a1=hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1., c67d00014c6ed04bf717813407dc89f8=hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8., d8016fe6cfb7745a592bb2fbc748686f=testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f.} 2024-12-03T19:00:59,961 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on 0de1d919473dc75d50db231a4234192c, 11506a0fb3ece4c0913a479f1e8f8d28, 1588230740, 903a9fc1a5dbd7bf3f74d3054afe53d5, 9b50dcf94758d2169683cf2c5fa74390, c67d00014c6ed04bf717813407dc89f8, d73667e828a0d4ccf3242e1c18ec0530, d8016fe6cfb7745a592bb2fbc748686f, e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:00:59,964 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnOpenScannerAsync/9b50dcf94758d2169683cf2c5fa74390/.tmp/testFamily/dcae8cb70cc54442a69e2594f937dda0 is 61, key is row-0/testFamily:testQualifier/1733252455826/Put/seqid=0 2024-12-03T19:00:59,968 INFO [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740/table/18519abbce5540d093daad0507bcd9ad, entries=20, sequenceid=56, filesize=6.5 K 2024-12-03T19:00:59,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741860_1036 (size=5049) 2024-12-03T19:00:59,970 INFO [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1010ms, sequenceid=56, compaction requested=false 2024-12-03T19:00:59,975 DEBUG [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-12-03T19:00:59,976 DEBUG [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T19:00:59,977 INFO [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T19:00:59,977 DEBUG [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733252458959Running coprocessor pre-close hooks at 1733252458959Disabling compacts and flushes for region at 1733252458959Disabling writes for close at 1733252458960 (+1 ms)Obtaining lock to block concurrent updates at 1733252458962 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1733252458962Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1733252458968 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733252458970 (+2 ms)Flushing 1588230740/info: creating writer at 1733252458971 (+1 ms)Flushing 1588230740/info: appending metadata at 1733252459012 (+41 ms)Flushing 1588230740/info: closing flushed file at 1733252459012Flushing 1588230740/ns: creating writer at 1733252459462 (+450 ms)Flushing 1588230740/ns: appending metadata at 1733252459483 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1733252459483Flushing 1588230740/table: creating writer at 1733252459499 (+16 ms)Flushing 1588230740/table: appending metadata at 1733252459522 (+23 ms)Flushing 1588230740/table: closing flushed file at 1733252459522Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73a873f8: reopening flushed file at 1733252459939 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bd2d5a2: reopening flushed file at 1733252459949 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e5a7ff9: reopening flushed file at 1733252459958 (+9 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1010ms, sequenceid=56, compaction requested=false at 1733252459970 (+12 ms)Writing region close event to WAL at 1733252459971 (+1 ms)Running coprocessor post-close hooks at 1733252459976 (+5 ms)Closed at 1733252459976 2024-12-03T19:00:59,977 DEBUG [RS_CLOSE_META-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T19:01:00,162 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on 0de1d919473dc75d50db231a4234192c, 11506a0fb3ece4c0913a479f1e8f8d28, 903a9fc1a5dbd7bf3f74d3054afe53d5, 9b50dcf94758d2169683cf2c5fa74390, c67d00014c6ed04bf717813407dc89f8, d73667e828a0d4ccf3242e1c18ec0530, d8016fe6cfb7745a592bb2fbc748686f, e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:01:00,362 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on 0de1d919473dc75d50db231a4234192c, 11506a0fb3ece4c0913a479f1e8f8d28, 903a9fc1a5dbd7bf3f74d3054afe53d5, 9b50dcf94758d2169683cf2c5fa74390, c67d00014c6ed04bf717813407dc89f8, d73667e828a0d4ccf3242e1c18ec0530, d8016fe6cfb7745a592bb2fbc748686f, e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:01:00,379 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnOpenScannerAsync/9b50dcf94758d2169683cf2c5fa74390/.tmp/testFamily/dcae8cb70cc54442a69e2594f937dda0 2024-12-03T19:01:00,395 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnOpenScannerAsync/9b50dcf94758d2169683cf2c5fa74390/.tmp/testFamily/dcae8cb70cc54442a69e2594f937dda0 as hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnOpenScannerAsync/9b50dcf94758d2169683cf2c5fa74390/testFamily/dcae8cb70cc54442a69e2594f937dda0 2024-12-03T19:01:00,403 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnOpenScannerAsync/9b50dcf94758d2169683cf2c5fa74390/testFamily/dcae8cb70cc54442a69e2594f937dda0, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T19:01:00,406 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9b50dcf94758d2169683cf2c5fa74390 in 459ms, sequenceid=8, compaction requested=false 2024-12-03T19:01:00,412 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnOpenScannerAsync/9b50dcf94758d2169683cf2c5fa74390/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T19:01:00,413 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390. 2024-12-03T19:01:00,414 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9b50dcf94758d2169683cf2c5fa74390: Waiting for close lock at 1733252459946Running coprocessor pre-close hooks at 1733252459946Disabling compacts and flushes for region at 1733252459946Disabling writes for close at 1733252459947 (+1 ms)Obtaining lock to block concurrent updates at 1733252459947Preparing flush snapshotting stores in 9b50dcf94758d2169683cf2c5fa74390 at 1733252459947Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733252459947Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390. at 1733252459948 (+1 ms)Flushing 9b50dcf94758d2169683cf2c5fa74390/testFamily: creating writer at 1733252459948Flushing 9b50dcf94758d2169683cf2c5fa74390/testFamily: appending metadata at 1733252459964 (+16 ms)Flushing 9b50dcf94758d2169683cf2c5fa74390/testFamily: closing flushed file at 1733252459964Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b41c461: reopening flushed file at 1733252460394 (+430 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9b50dcf94758d2169683cf2c5fa74390 in 459ms, sequenceid=8, compaction requested=false at 1733252460406 (+12 ms)Writing region close event to WAL at 1733252460407 (+1 ms)Running coprocessor post-close hooks at 1733252460413 (+6 ms)Closed at 1733252460413 2024-12-03T19:01:00,414 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733252452780.9b50dcf94758d2169683cf2c5fa74390. 2024-12-03T19:01:00,414 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d73667e828a0d4ccf3242e1c18ec0530, disabling compactions & flushes 2024-12-03T19:01:00,414 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530. 2024-12-03T19:01:00,414 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530. 2024-12-03T19:01:00,414 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530. after waiting 0 ms 2024-12-03T19:01:00,414 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530. 2024-12-03T19:01:00,415 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d73667e828a0d4ccf3242e1c18ec0530 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T19:01:00,436 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnOpenScanner/d73667e828a0d4ccf3242e1c18ec0530/.tmp/testFamily/3bc3a37766794931b67ce47d3f30b4fa is 61, key is row-0/testFamily:testQualifier/1733252402040/Put/seqid=0 2024-12-03T19:01:00,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741861_1037 (size=5049) 2024-12-03T19:01:00,563 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on 0de1d919473dc75d50db231a4234192c, 11506a0fb3ece4c0913a479f1e8f8d28, 903a9fc1a5dbd7bf3f74d3054afe53d5, c67d00014c6ed04bf717813407dc89f8, d73667e828a0d4ccf3242e1c18ec0530, d8016fe6cfb7745a592bb2fbc748686f, e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:01:00,763 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on 0de1d919473dc75d50db231a4234192c, 11506a0fb3ece4c0913a479f1e8f8d28, 903a9fc1a5dbd7bf3f74d3054afe53d5, c67d00014c6ed04bf717813407dc89f8, d73667e828a0d4ccf3242e1c18ec0530, d8016fe6cfb7745a592bb2fbc748686f, e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:01:00,844 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnOpenScanner/d73667e828a0d4ccf3242e1c18ec0530/.tmp/testFamily/3bc3a37766794931b67ce47d3f30b4fa 2024-12-03T19:01:00,892 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnOpenScanner/d73667e828a0d4ccf3242e1c18ec0530/.tmp/testFamily/3bc3a37766794931b67ce47d3f30b4fa as hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnOpenScanner/d73667e828a0d4ccf3242e1c18ec0530/testFamily/3bc3a37766794931b67ce47d3f30b4fa 2024-12-03T19:01:00,904 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnOpenScanner/d73667e828a0d4ccf3242e1c18ec0530/testFamily/3bc3a37766794931b67ce47d3f30b4fa, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T19:01:00,905 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d73667e828a0d4ccf3242e1c18ec0530 in 491ms, sequenceid=8, compaction requested=false 2024-12-03T19:01:00,910 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnOpenScanner/d73667e828a0d4ccf3242e1c18ec0530/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T19:01:00,911 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530. 2024-12-03T19:01:00,911 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d73667e828a0d4ccf3242e1c18ec0530: Waiting for close lock at 1733252460414Running coprocessor pre-close hooks at 1733252460414Disabling compacts and flushes for region at 1733252460414Disabling writes for close at 1733252460414Obtaining lock to block concurrent updates at 1733252460415 (+1 ms)Preparing flush snapshotting stores in d73667e828a0d4ccf3242e1c18ec0530 at 1733252460415Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733252460415Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530. at 1733252460416 (+1 ms)Flushing d73667e828a0d4ccf3242e1c18ec0530/testFamily: creating writer at 1733252460416Flushing d73667e828a0d4ccf3242e1c18ec0530/testFamily: appending metadata at 1733252460436 (+20 ms)Flushing d73667e828a0d4ccf3242e1c18ec0530/testFamily: closing flushed file at 1733252460436Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4175176a: reopening flushed file at 1733252460890 (+454 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d73667e828a0d4ccf3242e1c18ec0530 in 491ms, sequenceid=8, compaction requested=false at 1733252460905 (+15 ms)Writing region close event to WAL at 1733252460906 (+1 ms)Running coprocessor post-close hooks at 1733252460911 (+5 ms)Closed at 1733252460911 2024-12-03T19:01:00,911 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733252401003.d73667e828a0d4ccf3242e1c18ec0530. 2024-12-03T19:01:00,911 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0de1d919473dc75d50db231a4234192c, disabling compactions & flushes 2024-12-03T19:01:00,911 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c. 2024-12-03T19:01:00,911 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c. 2024-12-03T19:01:00,911 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c. after waiting 0 ms 2024-12-03T19:01:00,911 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c. 2024-12-03T19:01:00,912 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0de1d919473dc75d50db231a4234192c 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T19:01:00,926 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnNextAsync/0de1d919473dc75d50db231a4234192c/.tmp/testFamily/afeb9e985f2f44fda297ec7be9d5b965 is 61, key is row-0/testFamily:testQualifier/1733252427435/Put/seqid=0 2024-12-03T19:01:00,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741862_1038 (size=5049) 2024-12-03T19:01:00,963 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-12-03T19:01:00,964 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1325): Online Regions={0de1d919473dc75d50db231a4234192c=testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c., 11506a0fb3ece4c0913a479f1e8f8d28=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28., 903a9fc1a5dbd7bf3f74d3054afe53d5=testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5., e2eb87f7aa3da3d8944542aebd8f95a1=hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1., c67d00014c6ed04bf717813407dc89f8=hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8., d8016fe6cfb7745a592bb2fbc748686f=testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f.} 2024-12-03T19:01:00,964 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on 0de1d919473dc75d50db231a4234192c, 11506a0fb3ece4c0913a479f1e8f8d28, 903a9fc1a5dbd7bf3f74d3054afe53d5, c67d00014c6ed04bf717813407dc89f8, d8016fe6cfb7745a592bb2fbc748686f, e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:01:01,164 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on 0de1d919473dc75d50db231a4234192c, 11506a0fb3ece4c0913a479f1e8f8d28, 903a9fc1a5dbd7bf3f74d3054afe53d5, c67d00014c6ed04bf717813407dc89f8, d8016fe6cfb7745a592bb2fbc748686f, e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:01:01,333 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnNextAsync/0de1d919473dc75d50db231a4234192c/.tmp/testFamily/afeb9e985f2f44fda297ec7be9d5b965 2024-12-03T19:01:01,349 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnNextAsync/0de1d919473dc75d50db231a4234192c/.tmp/testFamily/afeb9e985f2f44fda297ec7be9d5b965 as hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnNextAsync/0de1d919473dc75d50db231a4234192c/testFamily/afeb9e985f2f44fda297ec7be9d5b965 2024-12-03T19:01:01,358 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnNextAsync/0de1d919473dc75d50db231a4234192c/testFamily/afeb9e985f2f44fda297ec7be9d5b965, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T19:01:01,360 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0de1d919473dc75d50db231a4234192c in 448ms, sequenceid=8, compaction requested=false 2024-12-03T19:01:01,364 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on 0de1d919473dc75d50db231a4234192c, 11506a0fb3ece4c0913a479f1e8f8d28, 903a9fc1a5dbd7bf3f74d3054afe53d5, c67d00014c6ed04bf717813407dc89f8, d8016fe6cfb7745a592bb2fbc748686f, e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:01:01,364 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnNextAsync/0de1d919473dc75d50db231a4234192c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T19:01:01,365 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c. 2024-12-03T19:01:01,365 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0de1d919473dc75d50db231a4234192c: Waiting for close lock at 1733252460911Running coprocessor pre-close hooks at 1733252460911Disabling compacts and flushes for region at 1733252460911Disabling writes for close at 1733252460911Obtaining lock to block concurrent updates at 1733252460912 (+1 ms)Preparing flush snapshotting stores in 0de1d919473dc75d50db231a4234192c at 1733252460912Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733252460912Flushing stores of testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c. at 1733252460912Flushing 0de1d919473dc75d50db231a4234192c/testFamily: creating writer at 1733252460912Flushing 0de1d919473dc75d50db231a4234192c/testFamily: appending metadata at 1733252460926 (+14 ms)Flushing 0de1d919473dc75d50db231a4234192c/testFamily: closing flushed file at 1733252460926Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41588a4f: reopening flushed file at 1733252461348 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0de1d919473dc75d50db231a4234192c in 448ms, sequenceid=8, compaction requested=false at 1733252461360 (+12 ms)Writing region close event to WAL at 1733252461360Running coprocessor post-close hooks at 1733252461365 (+5 ms)Closed at 1733252461365 2024-12-03T19:01:01,365 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1733252424365.0de1d919473dc75d50db231a4234192c. 2024-12-03T19:01:01,366 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 11506a0fb3ece4c0913a479f1e8f8d28, disabling compactions & flushes 2024-12-03T19:01:01,366 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28. 2024-12-03T19:01:01,366 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28. 2024-12-03T19:01:01,366 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28. after waiting 0 ms 2024-12-03T19:01:01,366 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28. 2024-12-03T19:01:01,366 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 11506a0fb3ece4c0913a479f1e8f8d28 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T19:01:01,381 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/11506a0fb3ece4c0913a479f1e8f8d28/.tmp/testFamily/35d5ac448f924968aaceb2d18a2c5648 is 61, key is row-0/testFamily:testQualifier/1733252437614/Put/seqid=0 2024-12-03T19:01:01,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741863_1039 (size=5049) 2024-12-03T19:01:01,565 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on 11506a0fb3ece4c0913a479f1e8f8d28, 903a9fc1a5dbd7bf3f74d3054afe53d5, c67d00014c6ed04bf717813407dc89f8, d8016fe6cfb7745a592bb2fbc748686f, e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:01:01,765 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on 11506a0fb3ece4c0913a479f1e8f8d28, 903a9fc1a5dbd7bf3f74d3054afe53d5, c67d00014c6ed04bf717813407dc89f8, d8016fe6cfb7745a592bb2fbc748686f, e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:01:01,788 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/11506a0fb3ece4c0913a479f1e8f8d28/.tmp/testFamily/35d5ac448f924968aaceb2d18a2c5648 2024-12-03T19:01:01,802 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/11506a0fb3ece4c0913a479f1e8f8d28/.tmp/testFamily/35d5ac448f924968aaceb2d18a2c5648 as hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/11506a0fb3ece4c0913a479f1e8f8d28/testFamily/35d5ac448f924968aaceb2d18a2c5648 2024-12-03T19:01:01,811 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/11506a0fb3ece4c0913a479f1e8f8d28/testFamily/35d5ac448f924968aaceb2d18a2c5648, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T19:01:01,812 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 11506a0fb3ece4c0913a479f1e8f8d28 in 446ms, sequenceid=8, compaction requested=false 2024-12-03T19:01:01,817 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/11506a0fb3ece4c0913a479f1e8f8d28/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T19:01:01,818 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28. 2024-12-03T19:01:01,818 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 11506a0fb3ece4c0913a479f1e8f8d28: Waiting for close lock at 1733252461366Running coprocessor pre-close hooks at 1733252461366Disabling compacts and flushes for region at 1733252461366Disabling writes for close at 1733252461366Obtaining lock to block concurrent updates at 1733252461366Preparing flush snapshotting stores in 11506a0fb3ece4c0913a479f1e8f8d28 at 1733252461366Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733252461366Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28. at 1733252461367 (+1 ms)Flushing 11506a0fb3ece4c0913a479f1e8f8d28/testFamily: creating writer at 1733252461367Flushing 11506a0fb3ece4c0913a479f1e8f8d28/testFamily: appending metadata at 1733252461381 (+14 ms)Flushing 11506a0fb3ece4c0913a479f1e8f8d28/testFamily: closing flushed file at 1733252461381Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7963ab71: reopening flushed file at 1733252461800 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 11506a0fb3ece4c0913a479f1e8f8d28 in 446ms, sequenceid=8, compaction requested=false at 1733252461812 (+12 ms)Writing region close event to WAL at 1733252461813 (+1 ms)Running coprocessor post-close hooks at 1733252461818 (+5 ms)Closed at 1733252461818 2024-12-03T19:01:01,819 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733252434563.11506a0fb3ece4c0913a479f1e8f8d28. 2024-12-03T19:01:01,819 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 903a9fc1a5dbd7bf3f74d3054afe53d5, disabling compactions & flushes 2024-12-03T19:01:01,819 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5. 2024-12-03T19:01:01,819 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5. 2024-12-03T19:01:01,819 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5. after waiting 0 ms 2024-12-03T19:01:01,819 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5. 2024-12-03T19:01:01,819 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 903a9fc1a5dbd7bf3f74d3054afe53d5 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T19:01:01,833 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnNext/903a9fc1a5dbd7bf3f74d3054afe53d5/.tmp/testFamily/74727ad862a44d6496655af1e4ace434 is 61, key is row-0/testFamily:testQualifier/1733252431516/Put/seqid=0 2024-12-03T19:01:01,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741864_1040 (size=5049) 2024-12-03T19:01:01,965 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-12-03T19:01:01,966 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1325): Online Regions={903a9fc1a5dbd7bf3f74d3054afe53d5=testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5., e2eb87f7aa3da3d8944542aebd8f95a1=hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1., c67d00014c6ed04bf717813407dc89f8=hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8., d8016fe6cfb7745a592bb2fbc748686f=testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f.} 2024-12-03T19:01:01,966 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on 903a9fc1a5dbd7bf3f74d3054afe53d5, c67d00014c6ed04bf717813407dc89f8, d8016fe6cfb7745a592bb2fbc748686f, e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:01:02,166 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on 903a9fc1a5dbd7bf3f74d3054afe53d5, c67d00014c6ed04bf717813407dc89f8, d8016fe6cfb7745a592bb2fbc748686f, e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:01:02,240 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnNext/903a9fc1a5dbd7bf3f74d3054afe53d5/.tmp/testFamily/74727ad862a44d6496655af1e4ace434 2024-12-03T19:01:02,254 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnNext/903a9fc1a5dbd7bf3f74d3054afe53d5/.tmp/testFamily/74727ad862a44d6496655af1e4ace434 as hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnNext/903a9fc1a5dbd7bf3f74d3054afe53d5/testFamily/74727ad862a44d6496655af1e4ace434 2024-12-03T19:01:02,264 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnNext/903a9fc1a5dbd7bf3f74d3054afe53d5/testFamily/74727ad862a44d6496655af1e4ace434, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T19:01:02,266 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 903a9fc1a5dbd7bf3f74d3054afe53d5 in 447ms, sequenceid=8, compaction requested=false 2024-12-03T19:01:02,271 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnNext/903a9fc1a5dbd7bf3f74d3054afe53d5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T19:01:02,272 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5. 2024-12-03T19:01:02,272 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 903a9fc1a5dbd7bf3f74d3054afe53d5: Waiting for close lock at 1733252461819Running coprocessor pre-close hooks at 1733252461819Disabling compacts and flushes for region at 1733252461819Disabling writes for close at 1733252461819Obtaining lock to block concurrent updates at 1733252461819Preparing flush snapshotting stores in 903a9fc1a5dbd7bf3f74d3054afe53d5 at 1733252461819Finished memstore snapshotting testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733252461819Flushing stores of testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5. at 1733252461820 (+1 ms)Flushing 903a9fc1a5dbd7bf3f74d3054afe53d5/testFamily: creating writer at 1733252461820Flushing 903a9fc1a5dbd7bf3f74d3054afe53d5/testFamily: appending metadata at 1733252461833 (+13 ms)Flushing 903a9fc1a5dbd7bf3f74d3054afe53d5/testFamily: closing flushed file at 1733252461833Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a2456dc: reopening flushed file at 1733252462252 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 903a9fc1a5dbd7bf3f74d3054afe53d5 in 447ms, sequenceid=8, compaction requested=false at 1733252462266 (+14 ms)Writing region close event to WAL at 1733252462267 (+1 ms)Running coprocessor post-close hooks at 1733252462272 (+5 ms)Closed at 1733252462272 2024-12-03T19:01:02,273 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1733252430482.903a9fc1a5dbd7bf3f74d3054afe53d5. 2024-12-03T19:01:02,273 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e2eb87f7aa3da3d8944542aebd8f95a1, disabling compactions & flushes 2024-12-03T19:01:02,273 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1. 2024-12-03T19:01:02,273 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1. 2024-12-03T19:01:02,273 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1. after waiting 0 ms 2024-12-03T19:01:02,273 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1. 2024-12-03T19:01:02,273 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e2eb87f7aa3da3d8944542aebd8f95a1 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T19:01:02,290 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnNext/e2eb87f7aa3da3d8944542aebd8f95a1/.tmp/testFamily/4a1e190acb11422f96755b63390b4cf7 is 61, key is row-0/testFamily:testQualifier/1733252446739/Put/seqid=0 2024-12-03T19:01:02,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741865_1041 (size=5049) 2024-12-03T19:01:02,367 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on c67d00014c6ed04bf717813407dc89f8, d8016fe6cfb7745a592bb2fbc748686f, e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:01:02,567 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on c67d00014c6ed04bf717813407dc89f8, d8016fe6cfb7745a592bb2fbc748686f, e2eb87f7aa3da3d8944542aebd8f95a1 2024-12-03T19:01:02,697 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnNext/e2eb87f7aa3da3d8944542aebd8f95a1/.tmp/testFamily/4a1e190acb11422f96755b63390b4cf7 2024-12-03T19:01:02,712 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnNext/e2eb87f7aa3da3d8944542aebd8f95a1/.tmp/testFamily/4a1e190acb11422f96755b63390b4cf7 as hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnNext/e2eb87f7aa3da3d8944542aebd8f95a1/testFamily/4a1e190acb11422f96755b63390b4cf7 2024-12-03T19:01:02,719 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnNext/e2eb87f7aa3da3d8944542aebd8f95a1/testFamily/4a1e190acb11422f96755b63390b4cf7, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T19:01:02,720 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e2eb87f7aa3da3d8944542aebd8f95a1 in 447ms, sequenceid=8, compaction requested=false 2024-12-03T19:01:02,725 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnNext/e2eb87f7aa3da3d8944542aebd8f95a1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T19:01:02,725 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1. 2024-12-03T19:01:02,725 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e2eb87f7aa3da3d8944542aebd8f95a1: Waiting for close lock at 1733252462273Running coprocessor pre-close hooks at 1733252462273Disabling compacts and flushes for region at 1733252462273Disabling writes for close at 1733252462273Obtaining lock to block concurrent updates at 1733252462273Preparing flush snapshotting stores in e2eb87f7aa3da3d8944542aebd8f95a1 at 1733252462273Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733252462273Flushing stores of hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1. at 1733252462274 (+1 ms)Flushing e2eb87f7aa3da3d8944542aebd8f95a1/testFamily: creating writer at 1733252462274Flushing e2eb87f7aa3da3d8944542aebd8f95a1/testFamily: appending metadata at 1733252462290 (+16 ms)Flushing e2eb87f7aa3da3d8944542aebd8f95a1/testFamily: closing flushed file at 1733252462290Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e5c90c4: reopening flushed file at 1733252462711 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e2eb87f7aa3da3d8944542aebd8f95a1 in 447ms, sequenceid=8, compaction requested=false at 1733252462720 (+9 ms)Writing region close event to WAL at 1733252462720Running coprocessor post-close hooks at 1733252462725 (+5 ms)Closed at 1733252462725 2024-12-03T19:01:02,726 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1733252443665.e2eb87f7aa3da3d8944542aebd8f95a1. 2024-12-03T19:01:02,726 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c67d00014c6ed04bf717813407dc89f8, disabling compactions & flushes 2024-12-03T19:01:02,726 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8. 2024-12-03T19:01:02,726 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8. 2024-12-03T19:01:02,726 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8. after waiting 0 ms 2024-12-03T19:01:02,726 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8. 2024-12-03T19:01:02,726 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c67d00014c6ed04bf717813407dc89f8 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T19:01:02,744 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnNextAsync/c67d00014c6ed04bf717813407dc89f8/.tmp/testFamily/2adaadb967db4fde915bab55b84a9774 is 61, key is row-0/testFamily:testQualifier/1733252411130/Put/seqid=0 2024-12-03T19:01:02,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741866_1042 (size=5049) 2024-12-03T19:01:02,767 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on c67d00014c6ed04bf717813407dc89f8, d8016fe6cfb7745a592bb2fbc748686f 2024-12-03T19:01:02,968 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-03T19:01:02,968 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1325): Online Regions={c67d00014c6ed04bf717813407dc89f8=hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8., d8016fe6cfb7745a592bb2fbc748686f=testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f.} 2024-12-03T19:01:02,968 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on c67d00014c6ed04bf717813407dc89f8, d8016fe6cfb7745a592bb2fbc748686f 2024-12-03T19:01:03,151 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnNextAsync/c67d00014c6ed04bf717813407dc89f8/.tmp/testFamily/2adaadb967db4fde915bab55b84a9774 2024-12-03T19:01:03,164 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnNextAsync/c67d00014c6ed04bf717813407dc89f8/.tmp/testFamily/2adaadb967db4fde915bab55b84a9774 as hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnNextAsync/c67d00014c6ed04bf717813407dc89f8/testFamily/2adaadb967db4fde915bab55b84a9774 2024-12-03T19:01:03,168 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on c67d00014c6ed04bf717813407dc89f8, d8016fe6cfb7745a592bb2fbc748686f 2024-12-03T19:01:03,173 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnNextAsync/c67d00014c6ed04bf717813407dc89f8/testFamily/2adaadb967db4fde915bab55b84a9774, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T19:01:03,174 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c67d00014c6ed04bf717813407dc89f8 in 448ms, sequenceid=8, compaction requested=false 2024-12-03T19:01:03,179 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/hbase/testMetaScanTimeoutOnNextAsync/c67d00014c6ed04bf717813407dc89f8/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T19:01:03,180 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8. 2024-12-03T19:01:03,180 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c67d00014c6ed04bf717813407dc89f8: Waiting for close lock at 1733252462726Running coprocessor pre-close hooks at 1733252462726Disabling compacts and flushes for region at 1733252462726Disabling writes for close at 1733252462726Obtaining lock to block concurrent updates at 1733252462726Preparing flush snapshotting stores in c67d00014c6ed04bf717813407dc89f8 at 1733252462726Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733252462726Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8. at 1733252462727 (+1 ms)Flushing c67d00014c6ed04bf717813407dc89f8/testFamily: creating writer at 1733252462727Flushing c67d00014c6ed04bf717813407dc89f8/testFamily: appending metadata at 1733252462744 (+17 ms)Flushing c67d00014c6ed04bf717813407dc89f8/testFamily: closing flushed file at 1733252462744Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@932030c: reopening flushed file at 1733252463162 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c67d00014c6ed04bf717813407dc89f8 in 448ms, sequenceid=8, compaction requested=false at 1733252463174 (+12 ms)Writing region close event to WAL at 1733252463175 (+1 ms)Running coprocessor post-close hooks at 1733252463179 (+4 ms)Closed at 1733252463179 2024-12-03T19:01:03,180 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733252408095.c67d00014c6ed04bf717813407dc89f8. 2024-12-03T19:01:03,180 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d8016fe6cfb7745a592bb2fbc748686f, disabling compactions & flushes 2024-12-03T19:01:03,180 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f. 2024-12-03T19:01:03,180 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f. 2024-12-03T19:01:03,180 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f. after waiting 0 ms 2024-12-03T19:01:03,180 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f. 2024-12-03T19:01:03,180 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d8016fe6cfb7745a592bb2fbc748686f 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T19:01:03,195 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnOpenScanner/d8016fe6cfb7745a592bb2fbc748686f/.tmp/testFamily/0170eb148c1141f28b6b1eff20cc06e3 is 61, key is row-0/testFamily:testQualifier/1733252421323/Put/seqid=0 2024-12-03T19:01:03,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741867_1043 (size=5049) 2024-12-03T19:01:03,368 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on d8016fe6cfb7745a592bb2fbc748686f 2024-12-03T19:01:03,569 DEBUG [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1351): Waiting on d8016fe6cfb7745a592bb2fbc748686f 2024-12-03T19:01:03,603 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnOpenScanner/d8016fe6cfb7745a592bb2fbc748686f/.tmp/testFamily/0170eb148c1141f28b6b1eff20cc06e3 2024-12-03T19:01:03,618 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnOpenScanner/d8016fe6cfb7745a592bb2fbc748686f/.tmp/testFamily/0170eb148c1141f28b6b1eff20cc06e3 as hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnOpenScanner/d8016fe6cfb7745a592bb2fbc748686f/testFamily/0170eb148c1141f28b6b1eff20cc06e3 2024-12-03T19:01:03,627 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnOpenScanner/d8016fe6cfb7745a592bb2fbc748686f/testFamily/0170eb148c1141f28b6b1eff20cc06e3, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T19:01:03,629 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d8016fe6cfb7745a592bb2fbc748686f in 449ms, sequenceid=8, compaction requested=false 2024-12-03T19:01:03,635 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/data/default/testNormalScanTimeoutOnOpenScanner/d8016fe6cfb7745a592bb2fbc748686f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T19:01:03,636 INFO [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f. 2024-12-03T19:01:03,636 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d8016fe6cfb7745a592bb2fbc748686f: Waiting for close lock at 1733252463180Running coprocessor pre-close hooks at 1733252463180Disabling compacts and flushes for region at 1733252463180Disabling writes for close at 1733252463180Obtaining lock to block concurrent updates at 1733252463180Preparing flush snapshotting stores in d8016fe6cfb7745a592bb2fbc748686f at 1733252463180Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733252463181 (+1 ms)Flushing stores of testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f. at 1733252463181Flushing d8016fe6cfb7745a592bb2fbc748686f/testFamily: creating writer at 1733252463181Flushing d8016fe6cfb7745a592bb2fbc748686f/testFamily: appending metadata at 1733252463195 (+14 ms)Flushing d8016fe6cfb7745a592bb2fbc748686f/testFamily: closing flushed file at 1733252463195Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@324c775: reopening flushed file at 1733252463617 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d8016fe6cfb7745a592bb2fbc748686f in 449ms, sequenceid=8, compaction requested=false at 1733252463629 (+12 ms)Writing region close event to WAL at 1733252463630 (+1 ms)Running coprocessor post-close hooks at 1733252463636 (+6 ms)Closed at 1733252463636 2024-12-03T19:01:03,636 DEBUG [RS_CLOSE_REGION-regionserver/db5a5ccf5be8:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1733252420288.d8016fe6cfb7745a592bb2fbc748686f. 2024-12-03T19:01:03,770 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(976): stopping server db5a5ccf5be8,32769,1733252334945; all regions closed. 2024-12-03T19:01:03,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741834_1010 (size=19479) 2024-12-03T19:01:03,788 DEBUG [RS:0;db5a5ccf5be8:32769 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/oldWALs 2024-12-03T19:01:03,789 INFO [RS:0;db5a5ccf5be8:32769 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL db5a5ccf5be8%2C32769%2C1733252334945.meta:.meta(num 1733252397323) 2024-12-03T19:01:03,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741833_1009 (size=20031) 2024-12-03T19:01:03,796 DEBUG [RS:0;db5a5ccf5be8:32769 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/oldWALs 2024-12-03T19:01:03,796 INFO [RS:0;db5a5ccf5be8:32769 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL db5a5ccf5be8%2C32769%2C1733252334945:(num 1733252396881) 2024-12-03T19:01:03,796 DEBUG [RS:0;db5a5ccf5be8:32769 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T19:01:03,796 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T19:01:03,796 INFO [RS:0;db5a5ccf5be8:32769 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T19:01:03,797 INFO [RS:0;db5a5ccf5be8:32769 {}] hbase.ChoreService(370): Chore service for: regionserver/db5a5ccf5be8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T19:01:03,797 INFO [RS:0;db5a5ccf5be8:32769 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T19:01:03,797 INFO [regionserver/db5a5ccf5be8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T19:01:03,799 INFO [RS:0;db5a5ccf5be8:32769 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32769 2024-12-03T19:01:03,823 INFO [regionserver/db5a5ccf5be8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T19:01:03,866 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32769-0x1019c8fba300001, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/db5a5ccf5be8,32769,1733252334945 2024-12-03T19:01:03,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T19:01:03,866 INFO [RS:0;db5a5ccf5be8:32769 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T19:01:03,869 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [db5a5ccf5be8,32769,1733252334945] 2024-12-03T19:01:03,887 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/db5a5ccf5be8,32769,1733252334945 already deleted, retry=false 2024-12-03T19:01:03,887 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; db5a5ccf5be8,32769,1733252334945 expired; onlineServers=0 2024-12-03T19:01:03,887 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'db5a5ccf5be8,33603,1733252334072' ***** 2024-12-03T19:01:03,887 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T19:01:03,887 INFO [M:0;db5a5ccf5be8:33603 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T19:01:03,888 INFO [M:0;db5a5ccf5be8:33603 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T19:01:03,888 DEBUG [M:0;db5a5ccf5be8:33603 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T19:01:03,889 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T19:01:03,889 DEBUG [M:0;db5a5ccf5be8:33603 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T19:01:03,889 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster-HFileCleaner.large.0-1733252336703 {}] cleaner.HFileCleaner(306): Exit Thread[master/db5a5ccf5be8:0:becomeActiveMaster-HFileCleaner.large.0-1733252336703,5,FailOnTimeoutGroup] 2024-12-03T19:01:03,889 DEBUG [master/db5a5ccf5be8:0:becomeActiveMaster-HFileCleaner.small.0-1733252336704 {}] cleaner.HFileCleaner(306): Exit Thread[master/db5a5ccf5be8:0:becomeActiveMaster-HFileCleaner.small.0-1733252336704,5,FailOnTimeoutGroup] 2024-12-03T19:01:03,889 INFO [M:0;db5a5ccf5be8:33603 {}] hbase.ChoreService(370): Chore service for: master/db5a5ccf5be8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T19:01:03,890 INFO [M:0;db5a5ccf5be8:33603 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T19:01:03,890 DEBUG [M:0;db5a5ccf5be8:33603 {}] master.HMaster(1795): Stopping service threads 2024-12-03T19:01:03,890 INFO [M:0;db5a5ccf5be8:33603 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T19:01:03,890 INFO [M:0;db5a5ccf5be8:33603 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T19:01:03,892 INFO [M:0;db5a5ccf5be8:33603 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T19:01:03,892 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T19:01:03,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T19:01:03,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T19:01:03,897 DEBUG [M:0;db5a5ccf5be8:33603 {}] zookeeper.ZKUtil(347): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T19:01:03,897 WARN [M:0;db5a5ccf5be8:33603 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T19:01:03,898 INFO [M:0;db5a5ccf5be8:33603 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/.lastflushedseqids 2024-12-03T19:01:03,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741868_1044 (size=712) 2024-12-03T19:01:03,977 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32769-0x1019c8fba300001, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T19:01:03,977 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32769-0x1019c8fba300001, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T19:01:03,978 INFO [RS:0;db5a5ccf5be8:32769 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T19:01:03,978 INFO [RS:0;db5a5ccf5be8:32769 {}] regionserver.HRegionServer(1031): Exiting; stopping=db5a5ccf5be8,32769,1733252334945; zookeeper connection closed. 2024-12-03T19:01:03,979 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5dfd5a2e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5dfd5a2e 2024-12-03T19:01:03,980 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-03T19:01:04,310 INFO [M:0;db5a5ccf5be8:33603 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T19:01:04,310 INFO [M:0;db5a5ccf5be8:33603 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T19:01:04,311 DEBUG [M:0;db5a5ccf5be8:33603 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T19:01:04,311 INFO [M:0;db5a5ccf5be8:33603 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T19:01:04,311 DEBUG [M:0;db5a5ccf5be8:33603 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T19:01:04,311 DEBUG [M:0;db5a5ccf5be8:33603 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T19:01:04,311 DEBUG [M:0;db5a5ccf5be8:33603 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T19:01:04,311 INFO [M:0;db5a5ccf5be8:33603 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.16 KB heapSize=191.18 KB 2024-12-03T19:01:04,331 DEBUG [M:0;db5a5ccf5be8:33603 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9b58fdb2fa024ea99fc4b890a499a160 is 82, key is hbase:meta,,1/info:regioninfo/1733252397421/Put/seqid=0 2024-12-03T19:01:04,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741869_1045 (size=5672) 2024-12-03T19:01:04,470 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T19:01:04,470 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-12-03T19:01:04,471 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-03T19:01:04,471 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-03T19:01:04,471 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-03T19:01:04,471 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-03T19:01:04,471 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T19:01:04,472 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-03T19:01:04,472 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-03T19:01:04,472 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-03T19:01:04,472 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-03T19:01:04,472 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-03T19:01:04,737 INFO [M:0;db5a5ccf5be8:33603 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9b58fdb2fa024ea99fc4b890a499a160 2024-12-03T19:01:04,767 DEBUG [M:0;db5a5ccf5be8:33603 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f8f427a255bf44a7bf0a448ee9361b68 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733252399101/Put/seqid=0 2024-12-03T19:01:04,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741870_1046 (size=14076) 2024-12-03T19:01:04,993 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T19:01:05,174 INFO [M:0;db5a5ccf5be8:33603 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.60 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f8f427a255bf44a7bf0a448ee9361b68 2024-12-03T19:01:05,186 INFO [M:0;db5a5ccf5be8:33603 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f8f427a255bf44a7bf0a448ee9361b68 2024-12-03T19:01:05,201 DEBUG [M:0;db5a5ccf5be8:33603 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6b01e73b8d5a4c418c716b4d8a2eb8db is 69, key is db5a5ccf5be8,32769,1733252334945/rs:state/1733252366695/Put/seqid=0 2024-12-03T19:01:05,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741871_1047 (size=5156) 2024-12-03T19:01:05,609 INFO [M:0;db5a5ccf5be8:33603 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6b01e73b8d5a4c418c716b4d8a2eb8db 2024-12-03T19:01:05,621 DEBUG [M:0;db5a5ccf5be8:33603 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9b58fdb2fa024ea99fc4b890a499a160 as hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9b58fdb2fa024ea99fc4b890a499a160 2024-12-03T19:01:05,628 INFO [M:0;db5a5ccf5be8:33603 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9b58fdb2fa024ea99fc4b890a499a160, entries=8, sequenceid=319, filesize=5.5 K 2024-12-03T19:01:05,629 DEBUG [M:0;db5a5ccf5be8:33603 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f8f427a255bf44a7bf0a448ee9361b68 as hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f8f427a255bf44a7bf0a448ee9361b68 2024-12-03T19:01:05,636 INFO [M:0;db5a5ccf5be8:33603 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f8f427a255bf44a7bf0a448ee9361b68 2024-12-03T19:01:05,636 INFO [M:0;db5a5ccf5be8:33603 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f8f427a255bf44a7bf0a448ee9361b68, entries=33, sequenceid=319, filesize=13.7 K 2024-12-03T19:01:05,637 DEBUG [M:0;db5a5ccf5be8:33603 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6b01e73b8d5a4c418c716b4d8a2eb8db as hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6b01e73b8d5a4c418c716b4d8a2eb8db 2024-12-03T19:01:05,644 INFO [M:0;db5a5ccf5be8:33603 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34585/user/jenkins/test-data/ebb512b0-3f33-266d-d3e0-e969fbcd678e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6b01e73b8d5a4c418c716b4d8a2eb8db, entries=1, sequenceid=319, filesize=5.0 K 2024-12-03T19:01:05,645 INFO [M:0;db5a5ccf5be8:33603 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.16 KB/167075, heapSize ~190.88 KB/195464, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1334ms, sequenceid=319, compaction requested=false 2024-12-03T19:01:05,646 INFO [M:0;db5a5ccf5be8:33603 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T19:01:05,647 DEBUG [M:0;db5a5ccf5be8:33603 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733252464310Disabling compacts and flushes for region at 1733252464310Disabling writes for close at 1733252464311 (+1 ms)Obtaining lock to block concurrent updates at 1733252464311Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733252464311Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=167075, getHeapSize=195704, getOffHeapSize=0, getCellsCount=366 at 1733252464311Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733252464312 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733252464312Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733252464330 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733252464330Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733252464748 (+418 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733252464766 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733252464766Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733252465186 (+420 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733252465201 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733252465201Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a35d17b: reopening flushed file at 1733252465620 (+419 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a5bd7a5: reopening flushed file at 1733252465628 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@780aaf1a: reopening flushed file at 1733252465636 (+8 ms)Finished flush of dataSize ~163.16 KB/167075, heapSize ~190.88 KB/195464, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1334ms, sequenceid=319, compaction requested=false at 1733252465645 (+9 ms)Writing region close event to WAL at 1733252465646 (+1 ms)Closed at 1733252465646 2024-12-03T19:01:05,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35847 is added to blk_1073741830_1006 (size=188417) 2024-12-03T19:01:05,650 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T19:01:05,650 INFO [M:0;db5a5ccf5be8:33603 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T19:01:05,651 INFO [M:0;db5a5ccf5be8:33603 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33603 2024-12-03T19:01:05,651 INFO [M:0;db5a5ccf5be8:33603 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T19:01:05,797 INFO [M:0;db5a5ccf5be8:33603 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T19:01:05,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T19:01:05,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33603-0x1019c8fba300000, quorum=127.0.0.1:59911, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T19:01:05,801 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5976fb41{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T19:01:05,804 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T19:01:05,804 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T19:01:05,804 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T19:01:05,804 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/hadoop.log.dir/,STOPPED} 2024-12-03T19:01:05,807 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T19:01:05,807 WARN [BP-43425679-172.17.0.2-1733252329908 heartbeating to localhost/127.0.0.1:34585 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T19:01:05,807 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T19:01:05,807 WARN [BP-43425679-172.17.0.2-1733252329908 heartbeating to localhost/127.0.0.1:34585 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-43425679-172.17.0.2-1733252329908 (Datanode Uuid 6aad047e-1653-4b8b-85b9-787e7b8b647b) service to localhost/127.0.0.1:34585 2024-12-03T19:01:05,809 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/cluster_ede8b3be-827f-ea87-d65f-2a8e90336803/data/data1/current/BP-43425679-172.17.0.2-1733252329908 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T19:01:05,809 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/cluster_ede8b3be-827f-ea87-d65f-2a8e90336803/data/data2/current/BP-43425679-172.17.0.2-1733252329908 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T19:01:05,809 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T19:01:05,816 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@eb30bc1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T19:01:05,817 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T19:01:05,817 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T19:01:05,817 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T19:01:05,817 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c500caaa-0998-d84a-96b9-8616d1199702/hadoop.log.dir/,STOPPED} 2024-12-03T19:01:05,823 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T19:01:05,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down