2024-12-09 20:28:26,738 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-09 20:28:26,765 main DEBUG Took 0.024287 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 20:28:26,766 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 20:28:26,766 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 20:28:26,767 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 20:28:26,769 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:28:26,778 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 20:28:26,793 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:28:26,795 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:28:26,796 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:28:26,796 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:28:26,797 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:28:26,798 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:28:26,799 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:28:26,799 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:28:26,800 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:28:26,800 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:28:26,801 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:28:26,801 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:28:26,802 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:28:26,802 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:28:26,803 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:28:26,803 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:28:26,804 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:28:26,804 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:28:26,805 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:28:26,805 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:28:26,806 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:28:26,806 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:28:26,807 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:28:26,807 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 20:28:26,808 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:28:26,808 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 20:28:26,810 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 20:28:26,812 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 20:28:26,814 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 20:28:26,815 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 20:28:26,816 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 20:28:26,817 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 20:28:26,828 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 20:28:26,836 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 20:28:26,838 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 20:28:26,839 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 20:28:26,839 main DEBUG createAppenders(={Console}) 2024-12-09 20:28:26,840 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-12-09 20:28:26,841 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-09 20:28:26,841 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-12-09 20:28:26,842 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 20:28:26,842 main DEBUG OutputStream closed 2024-12-09 20:28:26,843 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 20:28:26,843 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 20:28:26,844 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-12-09 20:28:26,935 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 20:28:26,938 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 20:28:26,940 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 20:28:26,941 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 20:28:26,942 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 20:28:26,942 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 20:28:26,943 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 20:28:26,943 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 20:28:26,943 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 20:28:26,944 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 20:28:26,944 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 20:28:26,944 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 20:28:26,945 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 20:28:26,945 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 20:28:26,945 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 20:28:26,946 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 20:28:26,946 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 20:28:26,947 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 20:28:26,950 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 20:28:26,950 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-12-09 20:28:26,951 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 20:28:26,951 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-12-09T20:28:26,966 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-12-09 20:28:26,969 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 20:28:26,969 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T20:28:27,302 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60 2024-12-09T20:28:27,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T20:28:27,360 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/cluster_0bfa8ede-61a6-2250-c0ec-ce485b9e5181, deleteOnExit=true 2024-12-09T20:28:27,360 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-09T20:28:27,362 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/test.cache.data in system properties and HBase conf 2024-12-09T20:28:27,362 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T20:28:27,363 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/hadoop.log.dir in system properties and HBase conf 2024-12-09T20:28:27,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T20:28:27,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T20:28:27,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-09T20:28:27,509 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T20:28:27,663 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T20:28:27,670 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T20:28:27,671 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T20:28:27,672 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T20:28:27,672 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T20:28:27,673 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T20:28:27,674 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T20:28:27,674 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T20:28:27,675 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T20:28:27,677 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T20:28:27,678 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/nfs.dump.dir in system properties and HBase conf 2024-12-09T20:28:27,679 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/java.io.tmpdir in system properties and HBase conf 2024-12-09T20:28:27,679 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T20:28:27,680 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T20:28:27,680 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T20:28:28,728 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T20:28:28,830 INFO [Time-limited test {}] log.Log(170): Logging initialized @2882ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T20:28:28,928 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T20:28:29,014 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T20:28:29,046 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T20:28:29,047 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T20:28:29,049 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T20:28:29,067 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T20:28:29,070 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/hadoop.log.dir/,AVAILABLE} 2024-12-09T20:28:29,071 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T20:28:29,309 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@eb30bc1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/java.io.tmpdir/jetty-localhost-46183-hadoop-hdfs-3_4_1-tests_jar-_-any-2566094847086304148/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T20:28:29,317 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:46183} 2024-12-09T20:28:29,318 INFO [Time-limited test {}] server.Server(415): Started @3371ms 2024-12-09T20:28:29,776 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T20:28:29,784 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T20:28:29,785 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T20:28:29,785 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T20:28:29,785 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T20:28:29,786 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/hadoop.log.dir/,AVAILABLE} 2024-12-09T20:28:29,787 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T20:28:29,923 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5976fb41{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/java.io.tmpdir/jetty-localhost-40559-hadoop-hdfs-3_4_1-tests_jar-_-any-2702919256216174511/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T20:28:29,924 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:40559} 2024-12-09T20:28:29,924 INFO [Time-limited test {}] server.Server(415): Started @3977ms 2024-12-09T20:28:29,985 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T20:28:30,521 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/cluster_0bfa8ede-61a6-2250-c0ec-ce485b9e5181/data/data1/current/BP-2093874798-172.17.0.2-1733776108440/current, will proceed with Du for space computation calculation, 2024-12-09T20:28:30,522 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/cluster_0bfa8ede-61a6-2250-c0ec-ce485b9e5181/data/data2/current/BP-2093874798-172.17.0.2-1733776108440/current, will proceed with Du for space computation calculation, 2024-12-09T20:28:30,572 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T20:28:30,643 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xad36d0926251f32a with lease ID 0x29762ba817764c71: Processing first storage report for DS-2ed5597d-178b-488e-8595-a06b3d757495 from datanode DatanodeRegistration(127.0.0.1:41477, datanodeUuid=b5d3d332-450a-40fa-aaa9-0c2f15522f69, infoPort=45381, infoSecurePort=0, ipcPort=46409, storageInfo=lv=-57;cid=testClusterID;nsid=1157941026;c=1733776108440) 2024-12-09T20:28:30,645 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xad36d0926251f32a with lease ID 0x29762ba817764c71: from storage DS-2ed5597d-178b-488e-8595-a06b3d757495 node DatanodeRegistration(127.0.0.1:41477, datanodeUuid=b5d3d332-450a-40fa-aaa9-0c2f15522f69, infoPort=45381, infoSecurePort=0, ipcPort=46409, storageInfo=lv=-57;cid=testClusterID;nsid=1157941026;c=1733776108440), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-09T20:28:30,646 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xad36d0926251f32a with lease ID 0x29762ba817764c71: Processing first storage report for DS-2882571b-24c7-4fc3-a03c-7b6e9f0b8a61 from datanode DatanodeRegistration(127.0.0.1:41477, datanodeUuid=b5d3d332-450a-40fa-aaa9-0c2f15522f69, infoPort=45381, infoSecurePort=0, ipcPort=46409, storageInfo=lv=-57;cid=testClusterID;nsid=1157941026;c=1733776108440) 2024-12-09T20:28:30,646 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xad36d0926251f32a with lease ID 0x29762ba817764c71: from storage DS-2882571b-24c7-4fc3-a03c-7b6e9f0b8a61 node DatanodeRegistration(127.0.0.1:41477, datanodeUuid=b5d3d332-450a-40fa-aaa9-0c2f15522f69, infoPort=45381, infoSecurePort=0, ipcPort=46409, storageInfo=lv=-57;cid=testClusterID;nsid=1157941026;c=1733776108440), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-09T20:28:30,722 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60 2024-12-09T20:28:30,822 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/cluster_0bfa8ede-61a6-2250-c0ec-ce485b9e5181/zookeeper_0, clientPort=61058, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/cluster_0bfa8ede-61a6-2250-c0ec-ce485b9e5181/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/cluster_0bfa8ede-61a6-2250-c0ec-ce485b9e5181/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T20:28:30,837 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61058 2024-12-09T20:28:30,868 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T20:28:30,873 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T20:28:31,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741825_1001 (size=7) 2024-12-09T20:28:31,629 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05 with version=8 2024-12-09T20:28:31,630 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/hbase-staging 2024-12-09T20:28:31,748 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T20:28:32,020 INFO [Time-limited test {}] client.ConnectionUtils(128): master/46f7c943b322:0 server-side Connection retries=9 2024-12-09T20:28:32,031 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T20:28:32,032 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T20:28:32,036 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T20:28:32,036 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T20:28:32,036 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T20:28:32,183 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T20:28:32,247 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T20:28:32,256 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T20:28:32,260 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T20:28:32,288 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 28469 (auto-detected) 2024-12-09T20:28:32,289 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-09T20:28:32,307 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36137 2024-12-09T20:28:32,329 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36137 connecting to ZooKeeper ensemble=127.0.0.1:61058 2024-12-09T20:28:32,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:361370x0, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T20:28:32,366 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36137-0x10198e9df570000 connected 2024-12-09T20:28:32,398 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T20:28:32,401 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T20:28:32,411 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T20:28:32,416 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05, hbase.cluster.distributed=false 2024-12-09T20:28:32,448 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T20:28:32,453 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36137 2024-12-09T20:28:32,453 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36137 2024-12-09T20:28:32,454 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36137 2024-12-09T20:28:32,454 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36137 2024-12-09T20:28:32,455 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36137 2024-12-09T20:28:32,574 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/46f7c943b322:0 server-side Connection retries=9 2024-12-09T20:28:32,576 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T20:28:32,576 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T20:28:32,577 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T20:28:32,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T20:28:32,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T20:28:32,580 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T20:28:32,582 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T20:28:32,583 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37083 2024-12-09T20:28:32,585 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37083 connecting to ZooKeeper ensemble=127.0.0.1:61058 2024-12-09T20:28:32,586 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T20:28:32,590 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T20:28:32,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:370830x0, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T20:28:32,605 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:370830x0, quorum=127.0.0.1:61058, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T20:28:32,607 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37083-0x10198e9df570001 connected 2024-12-09T20:28:32,611 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T20:28:32,622 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T20:28:32,625 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37083-0x10198e9df570001, quorum=127.0.0.1:61058, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T20:28:32,631 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37083-0x10198e9df570001, quorum=127.0.0.1:61058, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T20:28:32,636 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37083 2024-12-09T20:28:32,639 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37083 2024-12-09T20:28:32,640 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37083 2024-12-09T20:28:32,644 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37083 2024-12-09T20:28:32,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37083 2024-12-09T20:28:32,670 DEBUG [M:0;46f7c943b322:36137 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;46f7c943b322:36137 2024-12-09T20:28:32,671 INFO [master/46f7c943b322:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/46f7c943b322,36137,1733776111826 2024-12-09T20:28:32,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37083-0x10198e9df570001, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T20:28:32,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T20:28:32,689 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/46f7c943b322,36137,1733776111826 2024-12-09T20:28:32,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37083-0x10198e9df570001, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T20:28:32,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:28:32,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37083-0x10198e9df570001, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:28:32,715 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T20:28:32,716 INFO [master/46f7c943b322:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/46f7c943b322,36137,1733776111826 from backup master directory 2024-12-09T20:28:32,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/46f7c943b322,36137,1733776111826 2024-12-09T20:28:32,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37083-0x10198e9df570001, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T20:28:32,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T20:28:32,723 WARN [master/46f7c943b322:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T20:28:32,723 INFO [master/46f7c943b322:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=46f7c943b322,36137,1733776111826 2024-12-09T20:28:32,727 INFO [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T20:28:32,730 INFO [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T20:28:32,810 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/hbase.id] with ID: 96b7b085-1258-4022-88a7-06e21a0f787b 2024-12-09T20:28:32,811 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/.tmp/hbase.id 2024-12-09T20:28:32,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741826_1002 (size=42) 2024-12-09T20:28:32,825 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/.tmp/hbase.id]:[hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/hbase.id] 2024-12-09T20:28:32,876 INFO [master/46f7c943b322:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T20:28:32,881 INFO [master/46f7c943b322:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T20:28:32,905 INFO [master/46f7c943b322:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-12-09T20:28:32,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37083-0x10198e9df570001, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:28:32,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:28:32,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741827_1003 (size=196) 2024-12-09T20:28:32,955 INFO [master/46f7c943b322:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:28:32,958 INFO [master/46f7c943b322:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T20:28:32,975 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:28:32,979 INFO [master/46f7c943b322:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T20:28:33,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741828_1004 (size=1189) 2024-12-09T20:28:33,029 INFO [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/data/master/store 2024-12-09T20:28:33,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741829_1005 (size=34) 2024-12-09T20:28:33,455 INFO [master/46f7c943b322:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T20:28:33,458 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:28:33,460 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T20:28:33,460 INFO [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T20:28:33,460 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T20:28:33,461 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T20:28:33,462 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T20:28:33,462 INFO [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T20:28:33,463 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733776113460Disabling compacts and flushes for region at 1733776113460Disabling writes for close at 1733776113462 (+2 ms)Writing region close event to WAL at 1733776113462Closed at 1733776113462 2024-12-09T20:28:33,465 WARN [master/46f7c943b322:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/data/master/store/.initializing 2024-12-09T20:28:33,465 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/WALs/46f7c943b322,36137,1733776111826 2024-12-09T20:28:33,474 INFO [master/46f7c943b322:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T20:28:33,491 INFO [master/46f7c943b322:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=46f7c943b322%2C36137%2C1733776111826, suffix=, logDir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/WALs/46f7c943b322,36137,1733776111826, archiveDir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/oldWALs, maxLogs=10 2024-12-09T20:28:33,516 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/WALs/46f7c943b322,36137,1733776111826/46f7c943b322%2C36137%2C1733776111826.1733776113496, exclude list is [], retry=0 2024-12-09T20:28:33,533 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41477,DS-2ed5597d-178b-488e-8595-a06b3d757495,DISK] 2024-12-09T20:28:33,536 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T20:28:33,573 INFO [master/46f7c943b322:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/WALs/46f7c943b322,36137,1733776111826/46f7c943b322%2C36137%2C1733776111826.1733776113496 2024-12-09T20:28:33,574 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45381:45381)] 2024-12-09T20:28:33,574 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:28:33,575 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:28:33,578 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:28:33,579 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:28:33,616 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:28:33,644 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T20:28:33,648 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:28:33,651 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T20:28:33,651 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:28:33,655 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T20:28:33,655 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:28:33,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:28:33,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:28:33,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T20:28:33,660 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:28:33,661 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:28:33,661 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:28:33,664 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T20:28:33,664 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:28:33,665 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:28:33,665 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:28:33,669 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:28:33,671 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:28:33,676 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:28:33,676 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:28:33,681 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T20:28:33,685 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T20:28:33,690 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T20:28:33,692 INFO [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72404471, jitterRate=0.078910693526268}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T20:28:33,698 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733776113591Initializing all the Stores at 1733776113593 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733776113593Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776113594 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776113594Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776113594Cleaning up temporary data from old regions at 1733776113677 (+83 ms)Region opened successfully at 1733776113698 (+21 ms) 2024-12-09T20:28:33,699 INFO [master/46f7c943b322:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T20:28:33,734 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6567b266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=46f7c943b322/172.17.0.2:0 2024-12-09T20:28:33,768 INFO [master/46f7c943b322:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T20:28:33,780 INFO [master/46f7c943b322:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T20:28:33,780 INFO [master/46f7c943b322:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T20:28:33,783 INFO [master/46f7c943b322:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T20:28:33,784 INFO [master/46f7c943b322:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T20:28:33,789 INFO [master/46f7c943b322:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-09T20:28:33,790 INFO [master/46f7c943b322:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T20:28:33,816 INFO [master/46f7c943b322:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T20:28:33,825 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T20:28:33,843 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T20:28:33,845 INFO [master/46f7c943b322:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T20:28:33,847 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T20:28:33,866 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T20:28:33,869 INFO [master/46f7c943b322:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T20:28:33,873 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T20:28:33,888 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T20:28:33,889 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T20:28:33,896 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T20:28:33,915 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T20:28:33,917 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T20:28:33,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37083-0x10198e9df570001, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T20:28:33,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37083-0x10198e9df570001, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:28:33,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T20:28:33,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:28:33,927 INFO [master/46f7c943b322:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=46f7c943b322,36137,1733776111826, sessionid=0x10198e9df570000, setting cluster-up flag (Was=false) 2024-12-09T20:28:33,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:28:33,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37083-0x10198e9df570001, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:28:33,947 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T20:28:33,949 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=46f7c943b322,36137,1733776111826 2024-12-09T20:28:33,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:28:33,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37083-0x10198e9df570001, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:28:33,964 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T20:28:33,966 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=46f7c943b322,36137,1733776111826 2024-12-09T20:28:33,974 INFO [master/46f7c943b322:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T20:28:34,055 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(746): ClusterId : 96b7b085-1258-4022-88a7-06e21a0f787b 2024-12-09T20:28:34,059 DEBUG [RS:0;46f7c943b322:37083 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T20:28:34,059 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T20:28:34,064 DEBUG [RS:0;46f7c943b322:37083 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T20:28:34,064 DEBUG [RS:0;46f7c943b322:37083 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T20:28:34,069 DEBUG [RS:0;46f7c943b322:37083 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T20:28:34,070 DEBUG [RS:0;46f7c943b322:37083 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b14d0d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=46f7c943b322/172.17.0.2:0 2024-12-09T20:28:34,071 INFO [master/46f7c943b322:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T20:28:34,078 INFO [master/46f7c943b322:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T20:28:34,084 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 46f7c943b322,36137,1733776111826 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T20:28:34,089 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;46f7c943b322:37083 2024-12-09T20:28:34,092 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/46f7c943b322:0, corePoolSize=5, maxPoolSize=5 2024-12-09T20:28:34,092 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/46f7c943b322:0, corePoolSize=5, maxPoolSize=5 2024-12-09T20:28:34,092 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/46f7c943b322:0, corePoolSize=5, maxPoolSize=5 2024-12-09T20:28:34,092 INFO [RS:0;46f7c943b322:37083 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T20:28:34,092 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/46f7c943b322:0, corePoolSize=5, maxPoolSize=5 2024-12-09T20:28:34,092 INFO [RS:0;46f7c943b322:37083 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T20:28:34,093 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/46f7c943b322:0, corePoolSize=10, maxPoolSize=10 2024-12-09T20:28:34,093 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T20:28:34,093 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/46f7c943b322:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:28:34,093 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/46f7c943b322:0, corePoolSize=2, maxPoolSize=2 2024-12-09T20:28:34,094 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/46f7c943b322:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:28:34,096 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(2659): reportForDuty to master=46f7c943b322,36137,1733776111826 with port=37083, startcode=1733776112533 2024-12-09T20:28:34,098 INFO [master/46f7c943b322:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733776144098 2024-12-09T20:28:34,100 INFO [master/46f7c943b322:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T20:28:34,101 INFO [master/46f7c943b322:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T20:28:34,102 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T20:28:34,103 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T20:28:34,105 INFO [master/46f7c943b322:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T20:28:34,105 INFO [master/46f7c943b322:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T20:28:34,106 INFO [master/46f7c943b322:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T20:28:34,106 INFO [master/46f7c943b322:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T20:28:34,107 INFO [master/46f7c943b322:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T20:28:34,110 INFO [master/46f7c943b322:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T20:28:34,111 DEBUG [RS:0;46f7c943b322:37083 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T20:28:34,111 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:28:34,111 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T20:28:34,112 INFO [master/46f7c943b322:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T20:28:34,112 INFO [master/46f7c943b322:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T20:28:34,115 INFO [master/46f7c943b322:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T20:28:34,115 INFO [master/46f7c943b322:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T20:28:34,117 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/46f7c943b322:0:becomeActiveMaster-HFileCleaner.large.0-1733776114116,5,FailOnTimeoutGroup] 2024-12-09T20:28:34,118 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/46f7c943b322:0:becomeActiveMaster-HFileCleaner.small.0-1733776114117,5,FailOnTimeoutGroup] 2024-12-09T20:28:34,118 INFO [master/46f7c943b322:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T20:28:34,118 INFO [master/46f7c943b322:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T20:28:34,119 INFO [master/46f7c943b322:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T20:28:34,120 INFO [master/46f7c943b322:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T20:28:34,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741831_1007 (size=1321) 2024-12-09T20:28:34,125 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T20:28:34,125 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05 2024-12-09T20:28:34,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741832_1008 (size=32) 2024-12-09T20:28:34,151 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60531, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T20:28:34,157 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36137 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 46f7c943b322,37083,1733776112533 2024-12-09T20:28:34,160 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36137 {}] master.ServerManager(517): Registering regionserver=46f7c943b322,37083,1733776112533 2024-12-09T20:28:34,175 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05 2024-12-09T20:28:34,176 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40965 2024-12-09T20:28:34,176 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T20:28:34,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T20:28:34,181 DEBUG [RS:0;46f7c943b322:37083 {}] zookeeper.ZKUtil(111): regionserver:37083-0x10198e9df570001, quorum=127.0.0.1:61058, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/46f7c943b322,37083,1733776112533 2024-12-09T20:28:34,182 WARN [RS:0;46f7c943b322:37083 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T20:28:34,182 INFO [RS:0;46f7c943b322:37083 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T20:28:34,182 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/WALs/46f7c943b322,37083,1733776112533 2024-12-09T20:28:34,184 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [46f7c943b322,37083,1733776112533] 2024-12-09T20:28:34,209 INFO [RS:0;46f7c943b322:37083 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T20:28:34,223 INFO [RS:0;46f7c943b322:37083 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T20:28:34,228 INFO [RS:0;46f7c943b322:37083 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T20:28:34,228 INFO [RS:0;46f7c943b322:37083 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T20:28:34,229 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T20:28:34,236 INFO [RS:0;46f7c943b322:37083 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T20:28:34,237 INFO [RS:0;46f7c943b322:37083 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T20:28:34,237 DEBUG [RS:0;46f7c943b322:37083 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/46f7c943b322:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:28:34,238 DEBUG [RS:0;46f7c943b322:37083 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/46f7c943b322:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:28:34,238 DEBUG [RS:0;46f7c943b322:37083 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:28:34,238 DEBUG [RS:0;46f7c943b322:37083 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/46f7c943b322:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:28:34,238 DEBUG [RS:0;46f7c943b322:37083 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/46f7c943b322:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:28:34,239 DEBUG [RS:0;46f7c943b322:37083 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/46f7c943b322:0, corePoolSize=2, maxPoolSize=2 2024-12-09T20:28:34,239 DEBUG [RS:0;46f7c943b322:37083 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/46f7c943b322:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:28:34,239 DEBUG [RS:0;46f7c943b322:37083 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/46f7c943b322:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:28:34,239 DEBUG [RS:0;46f7c943b322:37083 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/46f7c943b322:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:28:34,239 DEBUG [RS:0;46f7c943b322:37083 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/46f7c943b322:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:28:34,240 DEBUG [RS:0;46f7c943b322:37083 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/46f7c943b322:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:28:34,240 DEBUG [RS:0;46f7c943b322:37083 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/46f7c943b322:0, corePoolSize=1, maxPoolSize=1 2024-12-09T20:28:34,240 DEBUG [RS:0;46f7c943b322:37083 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/46f7c943b322:0, corePoolSize=3, maxPoolSize=3 2024-12-09T20:28:34,241 DEBUG [RS:0;46f7c943b322:37083 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/46f7c943b322:0, corePoolSize=3, maxPoolSize=3 2024-12-09T20:28:34,242 INFO [RS:0;46f7c943b322:37083 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T20:28:34,243 INFO [RS:0;46f7c943b322:37083 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T20:28:34,243 INFO [RS:0;46f7c943b322:37083 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T20:28:34,243 INFO [RS:0;46f7c943b322:37083 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T20:28:34,243 INFO [RS:0;46f7c943b322:37083 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T20:28:34,243 INFO [RS:0;46f7c943b322:37083 {}] hbase.ChoreService(168): Chore ScheduledChore name=46f7c943b322,37083,1733776112533-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T20:28:34,272 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T20:28:34,274 INFO [RS:0;46f7c943b322:37083 {}] hbase.ChoreService(168): Chore ScheduledChore name=46f7c943b322,37083,1733776112533-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T20:28:34,274 INFO [RS:0;46f7c943b322:37083 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T20:28:34,274 INFO [RS:0;46f7c943b322:37083 {}] regionserver.Replication(171): 46f7c943b322,37083,1733776112533 started 2024-12-09T20:28:34,298 INFO [RS:0;46f7c943b322:37083 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T20:28:34,299 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1482): Serving as 46f7c943b322,37083,1733776112533, RpcServer on 46f7c943b322/172.17.0.2:37083, sessionid=0x10198e9df570001 2024-12-09T20:28:34,300 DEBUG [RS:0;46f7c943b322:37083 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T20:28:34,300 DEBUG [RS:0;46f7c943b322:37083 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 46f7c943b322,37083,1733776112533 2024-12-09T20:28:34,300 DEBUG [RS:0;46f7c943b322:37083 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '46f7c943b322,37083,1733776112533' 2024-12-09T20:28:34,301 DEBUG [RS:0;46f7c943b322:37083 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T20:28:34,302 DEBUG [RS:0;46f7c943b322:37083 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T20:28:34,303 DEBUG [RS:0;46f7c943b322:37083 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T20:28:34,303 DEBUG [RS:0;46f7c943b322:37083 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T20:28:34,303 DEBUG [RS:0;46f7c943b322:37083 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 46f7c943b322,37083,1733776112533 2024-12-09T20:28:34,303 DEBUG [RS:0;46f7c943b322:37083 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '46f7c943b322,37083,1733776112533' 2024-12-09T20:28:34,303 DEBUG [RS:0;46f7c943b322:37083 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T20:28:34,304 DEBUG [RS:0;46f7c943b322:37083 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T20:28:34,305 DEBUG [RS:0;46f7c943b322:37083 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T20:28:34,305 INFO [RS:0;46f7c943b322:37083 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T20:28:34,305 INFO [RS:0;46f7c943b322:37083 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T20:28:34,554 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:28:34,557 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T20:28:34,560 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T20:28:34,560 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:28:34,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T20:28:34,562 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T20:28:34,564 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T20:28:34,564 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:28:34,565 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T20:28:34,565 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T20:28:34,568 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T20:28:34,568 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:28:34,569 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T20:28:34,569 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T20:28:34,571 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T20:28:34,571 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:28:34,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T20:28:34,573 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T20:28:34,574 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740 2024-12-09T20:28:34,575 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740 2024-12-09T20:28:34,579 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T20:28:34,579 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T20:28:34,580 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T20:28:34,583 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T20:28:34,588 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T20:28:34,589 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75122067, jitterRate=0.11940602958202362}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T20:28:34,591 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733776114554Initializing all the Stores at 1733776114557 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733776114557Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733776114557Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776114557Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733776114557Cleaning up temporary data from old regions at 1733776114579 (+22 ms)Region opened successfully at 1733776114591 (+12 ms) 2024-12-09T20:28:34,592 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T20:28:34,592 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T20:28:34,592 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T20:28:34,592 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T20:28:34,592 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T20:28:34,594 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T20:28:34,594 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733776114592Disabling compacts and flushes for region at 1733776114592Disabling writes for close at 1733776114592Writing region close event to WAL at 1733776114593 (+1 ms)Closed at 1733776114594 (+1 ms) 2024-12-09T20:28:34,599 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T20:28:34,599 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T20:28:34,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T20:28:34,621 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T20:28:34,625 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T20:28:34,776 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:35,777 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:36,777 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:37,778 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:38,779 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:39,779 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:40,780 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:41,781 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:42,244 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T20:28:42,244 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T20:28:42,781 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:43,782 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:44,783 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:45,783 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:46,784 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:47,785 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:48,785 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:49,786 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:50,787 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:51,787 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:52,788 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:53,789 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:54,789 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:55,790 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:56,791 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:57,791 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:58,792 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:28:59,793 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:29:00,718 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T20:29:00,793 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:29:01,794 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:29:02,795 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:29:03,796 WARN [46f7c943b322:36137 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T20:29:04,310 INFO [RS:0;46f7c943b322:37083 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T20:29:04,315 INFO [RS:0;46f7c943b322:37083 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=46f7c943b322%2C37083%2C1733776112533, suffix=, logDir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/WALs/46f7c943b322,37083,1733776112533, archiveDir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/oldWALs, maxLogs=32 2024-12-09T20:29:04,332 DEBUG [RS:0;46f7c943b322:37083 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/WALs/46f7c943b322,37083,1733776112533/46f7c943b322%2C37083%2C1733776112533.1733776144317, exclude list is [], retry=0 2024-12-09T20:29:04,337 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41477,DS-2ed5597d-178b-488e-8595-a06b3d757495,DISK] 2024-12-09T20:29:04,341 INFO [RS:0;46f7c943b322:37083 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/WALs/46f7c943b322,37083,1733776112533/46f7c943b322%2C37083%2C1733776112533.1733776144317 2024-12-09T20:29:04,341 DEBUG [RS:0;46f7c943b322:37083 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45381:45381)] 2024-12-09T20:29:04,549 DEBUG [46f7c943b322:36137 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-09T20:29:04,561 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:29:04,568 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46f7c943b322,37083,1733776112533, state=OPENING 2024-12-09T20:29:04,576 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T20:29:04,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37083-0x10198e9df570001, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:29:04,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:29:04,578 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T20:29:04,578 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T20:29:04,580 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T20:29:04,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=46f7c943b322,37083,1733776112533}] 2024-12-09T20:29:04,758 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T20:29:04,762 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48701, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T20:29:04,773 INFO [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T20:29:04,773 INFO [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T20:29:04,774 INFO [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T20:29:04,777 INFO [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=46f7c943b322%2C37083%2C1733776112533.meta, suffix=.meta, logDir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/WALs/46f7c943b322,37083,1733776112533, archiveDir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/oldWALs, maxLogs=32 2024-12-09T20:29:04,793 DEBUG [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/WALs/46f7c943b322,37083,1733776112533/46f7c943b322%2C37083%2C1733776112533.meta.1733776144778.meta, exclude list is [], retry=0 2024-12-09T20:29:04,796 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41477,DS-2ed5597d-178b-488e-8595-a06b3d757495,DISK] 2024-12-09T20:29:04,799 INFO [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/WALs/46f7c943b322,37083,1733776112533/46f7c943b322%2C37083%2C1733776112533.meta.1733776144778.meta 2024-12-09T20:29:04,800 DEBUG [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45381:45381)] 2024-12-09T20:29:04,800 DEBUG [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:29:04,802 DEBUG [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T20:29:04,805 DEBUG [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T20:29:04,809 INFO [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T20:29:04,814 DEBUG [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T20:29:04,814 DEBUG [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:29:04,815 DEBUG [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T20:29:04,815 DEBUG [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T20:29:04,818 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T20:29:04,819 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T20:29:04,820 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:04,820 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T20:29:04,821 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T20:29:04,822 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T20:29:04,822 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:04,823 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T20:29:04,823 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T20:29:04,825 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T20:29:04,825 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:04,826 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T20:29:04,826 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T20:29:04,827 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T20:29:04,827 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:04,828 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T20:29:04,828 DEBUG [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T20:29:04,830 DEBUG [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740 2024-12-09T20:29:04,833 DEBUG [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740 2024-12-09T20:29:04,836 DEBUG [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T20:29:04,836 DEBUG [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T20:29:04,837 DEBUG [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T20:29:04,839 DEBUG [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T20:29:04,841 INFO [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63105902, jitterRate=-0.0596487820148468}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T20:29:04,841 DEBUG [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T20:29:04,843 DEBUG [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733776144815Writing region info on filesystem at 1733776144815Initializing all the Stores at 1733776144817 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733776144817Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733776144818 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776144818Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733776144818Cleaning up temporary data from old regions at 1733776144836 (+18 ms)Running coprocessor post-open hooks at 1733776144841 (+5 ms)Region opened successfully at 1733776144843 (+2 ms) 2024-12-09T20:29:04,851 INFO [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733776144748 2024-12-09T20:29:04,862 DEBUG [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T20:29:04,863 INFO [RS_OPEN_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T20:29:04,864 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:29:04,866 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 46f7c943b322,37083,1733776112533, state=OPEN 2024-12-09T20:29:04,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37083-0x10198e9df570001, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T20:29:04,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T20:29:04,873 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T20:29:04,873 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T20:29:04,873 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=46f7c943b322,37083,1733776112533 2024-12-09T20:29:04,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T20:29:04,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=46f7c943b322,37083,1733776112533 in 292 msec 2024-12-09T20:29:04,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T20:29:04,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.2720 sec 2024-12-09T20:29:04,885 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T20:29:04,885 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T20:29:04,905 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:29:04,907 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=46f7c943b322,37083,1733776112533, seqNum=-1] 2024-12-09T20:29:04,928 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:29:04,931 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51313, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:29:04,952 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 30.9450 sec 2024-12-09T20:29:04,953 INFO [master/46f7c943b322:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733776144953, completionTime=-1 2024-12-09T20:29:04,956 INFO [master/46f7c943b322:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-09T20:29:04,957 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T20:29:04,991 INFO [master/46f7c943b322:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-09T20:29:04,991 INFO [master/46f7c943b322:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733776204991 2024-12-09T20:29:04,992 INFO [master/46f7c943b322:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733776264992 2024-12-09T20:29:04,992 INFO [master/46f7c943b322:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 35 msec 2024-12-09T20:29:04,994 INFO [master/46f7c943b322:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=46f7c943b322,36137,1733776111826-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T20:29:04,994 INFO [master/46f7c943b322:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=46f7c943b322,36137,1733776111826-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T20:29:04,994 INFO [master/46f7c943b322:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=46f7c943b322,36137,1733776111826-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T20:29:04,996 INFO [master/46f7c943b322:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-46f7c943b322:36137, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T20:29:04,997 INFO [master/46f7c943b322:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T20:29:04,997 INFO [master/46f7c943b322:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T20:29:05,003 DEBUG [master/46f7c943b322:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T20:29:05,031 INFO [master/46f7c943b322:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 32.306sec 2024-12-09T20:29:05,033 INFO [master/46f7c943b322:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T20:29:05,035 INFO [master/46f7c943b322:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T20:29:05,036 INFO [master/46f7c943b322:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T20:29:05,037 INFO [master/46f7c943b322:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T20:29:05,037 INFO [master/46f7c943b322:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T20:29:05,038 INFO [master/46f7c943b322:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=46f7c943b322,36137,1733776111826-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T20:29:05,039 INFO [master/46f7c943b322:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=46f7c943b322,36137,1733776111826-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T20:29:05,048 DEBUG [master/46f7c943b322:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T20:29:05,049 INFO [master/46f7c943b322:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T20:29:05,049 INFO [master/46f7c943b322:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=46f7c943b322,36137,1733776111826-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T20:29:05,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77804624, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:29:05,117 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T20:29:05,117 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T20:29:05,121 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 46f7c943b322,36137,-1 for getting cluster id 2024-12-09T20:29:05,125 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:29:05,140 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '96b7b085-1258-4022-88a7-06e21a0f787b' 2024-12-09T20:29:05,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:29:05,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "96b7b085-1258-4022-88a7-06e21a0f787b" 2024-12-09T20:29:05,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36fc7c52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:29:05,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [46f7c943b322,36137,-1] 2024-12-09T20:29:05,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:29:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:29:05,153 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:29:05,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fad6539, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:29:05,157 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:29:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=46f7c943b322,37083,1733776112533, seqNum=-1] 2024-12-09T20:29:05,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:29:05,167 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:29:05,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=46f7c943b322,36137,1733776111826 2024-12-09T20:29:05,189 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68214a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:29:05,189 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 46f7c943b322,36137,-1 for getting cluster id 2024-12-09T20:29:05,189 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T20:29:05,190 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '96b7b085-1258-4022-88a7-06e21a0f787b' 2024-12-09T20:29:05,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T20:29:05,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "96b7b085-1258-4022-88a7-06e21a0f787b" 2024-12-09T20:29:05,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d3b3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:29:05,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [46f7c943b322,36137,-1] 2024-12-09T20:29:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T20:29:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:29:05,194 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T20:29:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e4c23c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T20:29:05,212 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=213, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=182, ProcessCount=11, AvailableMemoryMB=5735 2024-12-09T20:29:05,221 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T20:29:05,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 46f7c943b322,36137,1733776111826 2024-12-09T20:29:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6d4f7a07 2024-12-09T20:29:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T20:29:05,233 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T20:29:05,239 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:29:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-12-09T20:29:05,252 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T20:29:05,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-12-09T20:29:05,255 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:05,258 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T20:29:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T20:29:05,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741835_1011 (size=428) 2024-12-09T20:29:05,673 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a81806387c6394fb5229ed8e4f795ee2, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05 2024-12-09T20:29:05,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741836_1012 (size=79) 2024-12-09T20:29:06,088 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:29:06,088 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing a81806387c6394fb5229ed8e4f795ee2, disabling compactions & flushes 2024-12-09T20:29:06,088 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2. 2024-12-09T20:29:06,088 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2. 2024-12-09T20:29:06,088 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2. after waiting 0 ms 2024-12-09T20:29:06,089 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2. 2024-12-09T20:29:06,089 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2. 2024-12-09T20:29:06,089 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for a81806387c6394fb5229ed8e4f795ee2: Waiting for close lock at 1733776146088Disabling compacts and flushes for region at 1733776146088Disabling writes for close at 1733776146089 (+1 ms)Writing region close event to WAL at 1733776146089Closed at 1733776146089 2024-12-09T20:29:06,091 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T20:29:06,096 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733776146091"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733776146091"}]},"ts":"1733776146091"} 2024-12-09T20:29:06,102 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T20:29:06,103 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T20:29:06,106 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776146103"}]},"ts":"1733776146103"} 2024-12-09T20:29:06,111 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-12-09T20:29:06,113 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=a81806387c6394fb5229ed8e4f795ee2, ASSIGN}] 2024-12-09T20:29:06,115 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=a81806387c6394fb5229ed8e4f795ee2, ASSIGN 2024-12-09T20:29:06,117 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=a81806387c6394fb5229ed8e4f795ee2, ASSIGN; state=OFFLINE, location=46f7c943b322,37083,1733776112533; forceNewPlan=false, retain=false 2024-12-09T20:29:06,269 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a81806387c6394fb5229ed8e4f795ee2, regionState=OPENING, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:29:06,274 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=a81806387c6394fb5229ed8e4f795ee2, ASSIGN because future has completed 2024-12-09T20:29:06,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a81806387c6394fb5229ed8e4f795ee2, server=46f7c943b322,37083,1733776112533}] 2024-12-09T20:29:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T20:29:06,435 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2. 2024-12-09T20:29:06,435 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => a81806387c6394fb5229ed8e4f795ee2, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:29:06,436 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync a81806387c6394fb5229ed8e4f795ee2 2024-12-09T20:29:06,436 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:29:06,436 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for a81806387c6394fb5229ed8e4f795ee2 2024-12-09T20:29:06,436 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for a81806387c6394fb5229ed8e4f795ee2 2024-12-09T20:29:06,439 INFO [StoreOpener-a81806387c6394fb5229ed8e4f795ee2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a81806387c6394fb5229ed8e4f795ee2 2024-12-09T20:29:06,441 INFO [StoreOpener-a81806387c6394fb5229ed8e4f795ee2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a81806387c6394fb5229ed8e4f795ee2 columnFamilyName testFamily 2024-12-09T20:29:06,441 DEBUG [StoreOpener-a81806387c6394fb5229ed8e4f795ee2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:06,442 INFO [StoreOpener-a81806387c6394fb5229ed8e4f795ee2-1 {}] regionserver.HStore(327): Store=a81806387c6394fb5229ed8e4f795ee2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:29:06,443 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for a81806387c6394fb5229ed8e4f795ee2 2024-12-09T20:29:06,444 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a81806387c6394fb5229ed8e4f795ee2 2024-12-09T20:29:06,445 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a81806387c6394fb5229ed8e4f795ee2 2024-12-09T20:29:06,445 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for a81806387c6394fb5229ed8e4f795ee2 2024-12-09T20:29:06,445 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for a81806387c6394fb5229ed8e4f795ee2 2024-12-09T20:29:06,448 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for a81806387c6394fb5229ed8e4f795ee2 2024-12-09T20:29:06,452 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a81806387c6394fb5229ed8e4f795ee2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T20:29:06,453 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened a81806387c6394fb5229ed8e4f795ee2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64197077, jitterRate=-0.043389007449150085}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T20:29:06,453 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a81806387c6394fb5229ed8e4f795ee2 2024-12-09T20:29:06,454 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for a81806387c6394fb5229ed8e4f795ee2: Running coprocessor pre-open hook at 1733776146436Writing region info on filesystem at 1733776146436Initializing all the Stores at 1733776146438 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776146438Cleaning up temporary data from old regions at 1733776146446 (+8 ms)Running coprocessor post-open hooks at 1733776146453 (+7 ms)Region opened successfully at 1733776146454 (+1 ms) 2024-12-09T20:29:06,456 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2., pid=6, masterSystemTime=1733776146428 2024-12-09T20:29:06,460 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2. 2024-12-09T20:29:06,460 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2. 2024-12-09T20:29:06,462 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a81806387c6394fb5229ed8e4f795ee2, regionState=OPEN, openSeqNum=2, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:29:06,465 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a81806387c6394fb5229ed8e4f795ee2, server=46f7c943b322,37083,1733776112533 because future has completed 2024-12-09T20:29:06,470 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T20:29:06,470 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a81806387c6394fb5229ed8e4f795ee2, server=46f7c943b322,37083,1733776112533 in 192 msec 2024-12-09T20:29:06,474 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T20:29:06,474 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=a81806387c6394fb5229ed8e4f795ee2, ASSIGN in 357 msec 2024-12-09T20:29:06,475 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T20:29:06,476 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776146475"}]},"ts":"1733776146475"} 2024-12-09T20:29:06,478 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-12-09T20:29:06,480 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T20:29:06,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2380 sec 2024-12-09T20:29:08,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T20:29:08,302 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-12-09T20:29:08,303 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-12-09T20:29:08,305 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T20:29:08,311 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-12-09T20:29:08,311 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T20:29:08,312 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-12-09T20:29:08,315 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T20:29:08,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=46f7c943b322,37083,1733776112533, seqNum=-1] 2024-12-09T20:29:08,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T20:29:08,321 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T20:29:08,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2., hostname=46f7c943b322,37083,1733776112533, seqNum=2] 2024-12-09T20:29:08,345 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T20:29:08,349 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-12-09T20:29:08,354 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-09T20:29:08,360 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-09T20:29:08,362 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-09T20:29:08,363 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-7227005361465589752 2024-12-09T20:29:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:59098 deadline: 1733776151362, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T20:29:08,403 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-09T20:29:08,405 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-09T20:29:08,406 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T20:29:08,409 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T20:29:08,410 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-7227005361465589750 2024-12-09T20:29:08,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:59098 deadline: 1733776151408, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T20:29:08,415 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T20:29:08,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-7227005361465589749 2024-12-09T20:29:08,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:59098 deadline: 1733776151414, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T20:29:08,420 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T20:29:08,421 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-7227005361465589748 2024-12-09T20:29:08,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:59098 deadline: 1733776151420, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T20:29:08,426 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T20:29:08,439 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=219 (was 213) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/46f7c943b322:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-543335734_22 at /127.0.0.1:36672 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=182 (was 182), ProcessCount=11 (was 11), AvailableMemoryMB=5718 (was 5735) 2024-12-09T20:29:08,450 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=219, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=182, ProcessCount=11, AvailableMemoryMB=5718 2024-12-09T20:29:08,455 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36137 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:29:08,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36137 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-12-09T20:29:08,461 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T20:29:08,461 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:08,461 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36137 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-12-09T20:29:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T20:29:08,463 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T20:29:08,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741837_1013 (size=414) 2024-12-09T20:29:08,876 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a911b4a9b1ae731dcfe55954949d9033, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05 2024-12-09T20:29:08,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741838_1014 (size=65) 2024-12-09T20:29:09,287 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:29:09,287 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing a911b4a9b1ae731dcfe55954949d9033, disabling compactions & flushes 2024-12-09T20:29:09,287 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033. 2024-12-09T20:29:09,287 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033. 2024-12-09T20:29:09,287 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033. after waiting 0 ms 2024-12-09T20:29:09,287 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033. 2024-12-09T20:29:09,287 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033. 2024-12-09T20:29:09,287 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for a911b4a9b1ae731dcfe55954949d9033: Waiting for close lock at 1733776149287Disabling compacts and flushes for region at 1733776149287Disabling writes for close at 1733776149287Writing region close event to WAL at 1733776149287Closed at 1733776149287 2024-12-09T20:29:09,290 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T20:29:09,290 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1733776149290"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733776149290"}]},"ts":"1733776149290"} 2024-12-09T20:29:09,294 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T20:29:09,296 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T20:29:09,296 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776149296"}]},"ts":"1733776149296"} 2024-12-09T20:29:09,299 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-09T20:29:09,300 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=a911b4a9b1ae731dcfe55954949d9033, ASSIGN}] 2024-12-09T20:29:09,302 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=a911b4a9b1ae731dcfe55954949d9033, ASSIGN 2024-12-09T20:29:09,303 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=a911b4a9b1ae731dcfe55954949d9033, ASSIGN; state=OFFLINE, location=46f7c943b322,37083,1733776112533; forceNewPlan=false, retain=false 2024-12-09T20:29:09,455 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=a911b4a9b1ae731dcfe55954949d9033, regionState=OPENING, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:29:09,459 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=a911b4a9b1ae731dcfe55954949d9033, ASSIGN because future has completed 2024-12-09T20:29:09,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure a911b4a9b1ae731dcfe55954949d9033, server=46f7c943b322,37083,1733776112533}] 2024-12-09T20:29:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T20:29:09,621 INFO [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033. 2024-12-09T20:29:09,621 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => a911b4a9b1ae731dcfe55954949d9033, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:29:09,621 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner a911b4a9b1ae731dcfe55954949d9033 2024-12-09T20:29:09,621 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:29:09,622 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for a911b4a9b1ae731dcfe55954949d9033 2024-12-09T20:29:09,622 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for a911b4a9b1ae731dcfe55954949d9033 2024-12-09T20:29:09,624 INFO [StoreOpener-a911b4a9b1ae731dcfe55954949d9033-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a911b4a9b1ae731dcfe55954949d9033 2024-12-09T20:29:09,626 INFO [StoreOpener-a911b4a9b1ae731dcfe55954949d9033-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a911b4a9b1ae731dcfe55954949d9033 columnFamilyName testFamily 2024-12-09T20:29:09,627 DEBUG [StoreOpener-a911b4a9b1ae731dcfe55954949d9033-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:09,627 INFO [StoreOpener-a911b4a9b1ae731dcfe55954949d9033-1 {}] regionserver.HStore(327): Store=a911b4a9b1ae731dcfe55954949d9033/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:29:09,628 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for a911b4a9b1ae731dcfe55954949d9033 2024-12-09T20:29:09,629 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnOpenScanner/a911b4a9b1ae731dcfe55954949d9033 2024-12-09T20:29:09,630 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnOpenScanner/a911b4a9b1ae731dcfe55954949d9033 2024-12-09T20:29:09,631 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for a911b4a9b1ae731dcfe55954949d9033 2024-12-09T20:29:09,631 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for a911b4a9b1ae731dcfe55954949d9033 2024-12-09T20:29:09,633 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for a911b4a9b1ae731dcfe55954949d9033 2024-12-09T20:29:09,637 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnOpenScanner/a911b4a9b1ae731dcfe55954949d9033/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T20:29:09,638 INFO [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened a911b4a9b1ae731dcfe55954949d9033; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67702495, jitterRate=0.008845791220664978}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T20:29:09,638 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a911b4a9b1ae731dcfe55954949d9033 2024-12-09T20:29:09,639 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for a911b4a9b1ae731dcfe55954949d9033: Running coprocessor pre-open hook at 1733776149622Writing region info on filesystem at 1733776149622Initializing all the Stores at 1733776149623 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776149624 (+1 ms)Cleaning up temporary data from old regions at 1733776149631 (+7 ms)Running coprocessor post-open hooks at 1733776149638 (+7 ms)Region opened successfully at 1733776149639 (+1 ms) 2024-12-09T20:29:09,641 INFO [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033., pid=9, masterSystemTime=1733776149614 2024-12-09T20:29:09,645 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033. 2024-12-09T20:29:09,645 INFO [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033. 2024-12-09T20:29:09,646 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=a911b4a9b1ae731dcfe55954949d9033, regionState=OPEN, openSeqNum=2, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:29:09,650 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure a911b4a9b1ae731dcfe55954949d9033, server=46f7c943b322,37083,1733776112533 because future has completed 2024-12-09T20:29:09,656 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-09T20:29:09,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure a911b4a9b1ae731dcfe55954949d9033, server=46f7c943b322,37083,1733776112533 in 192 msec 2024-12-09T20:29:09,660 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T20:29:09,660 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=a911b4a9b1ae731dcfe55954949d9033, ASSIGN in 356 msec 2024-12-09T20:29:09,661 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T20:29:09,662 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776149661"}]},"ts":"1733776149661"} 2024-12-09T20:29:09,665 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-09T20:29:09,666 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T20:29:09,669 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2110 sec 2024-12-09T20:29:10,345 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T20:29:10,376 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-12-09T20:29:10,378 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T20:29:10,379 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-12-09T20:29:11,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T20:29:11,482 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-12-09T20:29:11,482 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-09T20:29:11,482 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T20:29:11,488 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-09T20:29:11,488 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T20:29:11,488 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-12-09T20:29:11,492 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033., hostname=46f7c943b322,37083,1733776112533, seqNum=2] 2024-12-09T20:29:11,504 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T20:29:11,505 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-09T20:29:11,508 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-09T20:29:11,510 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37083 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-09T20:29:12,243 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T20:29:12,243 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T20:29:12,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T20:29:12,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T20:29:12,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-09T20:29:12,246 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-12-09T20:29:12,247 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-09T20:29:12,247 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-09T20:29:17,523 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033., hostname=46f7c943b322,37083,1733776112533, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033., hostname=46f7c943b322,37083,1733776112533, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=46f7c943b322:37083 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6012ms, rpcTimeout=5999ms 2024-12-09T20:29:17,524 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033., hostname=46f7c943b322,37083,1733776112533, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6012ms, rpcTimeout=5999ms 2024-12-09T20:29:17,524 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033., hostname=46f7c943b322,37083,1733776112533, seqNum=2 from cache 2024-12-09T20:29:17,537 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=222 (was 219) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=169 (was 182), ProcessCount=11 (was 11), AvailableMemoryMB=5695 (was 5718) 2024-12-09T20:29:17,546 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=169, ProcessCount=11, AvailableMemoryMB=5694 2024-12-09T20:29:17,549 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36137 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:29:17,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36137 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-12-09T20:29:17,552 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T20:29:17,553 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:17,553 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36137 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-12-09T20:29:17,554 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T20:29:17,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-09T20:29:17,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741839_1015 (size=412) 2024-12-09T20:29:17,965 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6e53f96f465f63d860346b347adf8ce9, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05 2024-12-09T20:29:17,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741840_1016 (size=63) 2024-12-09T20:29:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-12-09T20:29:18,375 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:29:18,375 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 6e53f96f465f63d860346b347adf8ce9, disabling compactions & flushes 2024-12-09T20:29:18,375 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9. 2024-12-09T20:29:18,375 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9. 2024-12-09T20:29:18,375 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9. after waiting 0 ms 2024-12-09T20:29:18,375 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9. 2024-12-09T20:29:18,375 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9. 2024-12-09T20:29:18,375 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6e53f96f465f63d860346b347adf8ce9: Waiting for close lock at 1733776158375Disabling compacts and flushes for region at 1733776158375Disabling writes for close at 1733776158375Writing region close event to WAL at 1733776158375Closed at 1733776158375 2024-12-09T20:29:18,377 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T20:29:18,378 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1733776158378"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733776158378"}]},"ts":"1733776158378"} 2024-12-09T20:29:18,381 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T20:29:18,382 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T20:29:18,382 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776158382"}]},"ts":"1733776158382"} 2024-12-09T20:29:18,385 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-09T20:29:18,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=6e53f96f465f63d860346b347adf8ce9, ASSIGN}] 2024-12-09T20:29:18,387 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=6e53f96f465f63d860346b347adf8ce9, ASSIGN 2024-12-09T20:29:18,388 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=6e53f96f465f63d860346b347adf8ce9, ASSIGN; state=OFFLINE, location=46f7c943b322,37083,1733776112533; forceNewPlan=false, retain=false 2024-12-09T20:29:18,539 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=6e53f96f465f63d860346b347adf8ce9, regionState=OPENING, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:29:18,543 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=6e53f96f465f63d860346b347adf8ce9, ASSIGN because future has completed 2024-12-09T20:29:18,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e53f96f465f63d860346b347adf8ce9, server=46f7c943b322,37083,1733776112533}] 2024-12-09T20:29:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-09T20:29:18,703 INFO [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9. 2024-12-09T20:29:18,703 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 6e53f96f465f63d860346b347adf8ce9, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:29:18,704 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 6e53f96f465f63d860346b347adf8ce9 2024-12-09T20:29:18,704 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:29:18,704 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 6e53f96f465f63d860346b347adf8ce9 2024-12-09T20:29:18,704 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 6e53f96f465f63d860346b347adf8ce9 2024-12-09T20:29:18,706 INFO [StoreOpener-6e53f96f465f63d860346b347adf8ce9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 6e53f96f465f63d860346b347adf8ce9 2024-12-09T20:29:18,709 INFO [StoreOpener-6e53f96f465f63d860346b347adf8ce9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e53f96f465f63d860346b347adf8ce9 columnFamilyName testFamily 2024-12-09T20:29:18,709 DEBUG [StoreOpener-6e53f96f465f63d860346b347adf8ce9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:18,709 INFO [StoreOpener-6e53f96f465f63d860346b347adf8ce9-1 {}] regionserver.HStore(327): Store=6e53f96f465f63d860346b347adf8ce9/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:29:18,710 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 6e53f96f465f63d860346b347adf8ce9 2024-12-09T20:29:18,711 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnNextAsync/6e53f96f465f63d860346b347adf8ce9 2024-12-09T20:29:18,711 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnNextAsync/6e53f96f465f63d860346b347adf8ce9 2024-12-09T20:29:18,712 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 6e53f96f465f63d860346b347adf8ce9 2024-12-09T20:29:18,712 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 6e53f96f465f63d860346b347adf8ce9 2024-12-09T20:29:18,714 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 6e53f96f465f63d860346b347adf8ce9 2024-12-09T20:29:18,717 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnNextAsync/6e53f96f465f63d860346b347adf8ce9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T20:29:18,718 INFO [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 6e53f96f465f63d860346b347adf8ce9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63540728, jitterRate=-0.0531693696975708}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T20:29:18,718 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6e53f96f465f63d860346b347adf8ce9 2024-12-09T20:29:18,719 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 6e53f96f465f63d860346b347adf8ce9: Running coprocessor pre-open hook at 1733776158704Writing region info on filesystem at 1733776158704Initializing all the Stores at 1733776158706 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776158706Cleaning up temporary data from old regions at 1733776158712 (+6 ms)Running coprocessor post-open hooks at 1733776158718 (+6 ms)Region opened successfully at 1733776158719 (+1 ms) 2024-12-09T20:29:18,720 INFO [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9., pid=12, masterSystemTime=1733776158697 2024-12-09T20:29:18,724 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9. 2024-12-09T20:29:18,724 INFO [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9. 2024-12-09T20:29:18,725 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=6e53f96f465f63d860346b347adf8ce9, regionState=OPEN, openSeqNum=2, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:29:18,728 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e53f96f465f63d860346b347adf8ce9, server=46f7c943b322,37083,1733776112533 because future has completed 2024-12-09T20:29:18,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-09T20:29:18,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 6e53f96f465f63d860346b347adf8ce9, server=46f7c943b322,37083,1733776112533 in 186 msec 2024-12-09T20:29:18,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-09T20:29:18,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=6e53f96f465f63d860346b347adf8ce9, ASSIGN in 348 msec 2024-12-09T20:29:18,737 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T20:29:18,737 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776158737"}]},"ts":"1733776158737"} 2024-12-09T20:29:18,740 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-09T20:29:18,741 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T20:29:18,744 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.1920 sec 2024-12-09T20:29:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-09T20:29:20,572 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-12-09T20:29:20,572 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-09T20:29:20,572 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T20:29:20,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-09T20:29:20,577 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T20:29:20,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-12-09T20:29:20,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9., hostname=46f7c943b322,37083,1733776112533, seqNum=2] 2024-12-09T20:29:20,593 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T20:29:20,594 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-09T20:29:20,597 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-09T20:29:20,599 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-09T20:29:20,600 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-09T20:29:20,601 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37083 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-09T20:29:22,243 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-09T20:29:22,243 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-09T20:29:22,381 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-12-09T20:29:24,231 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T20:29:26,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37083 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7227005361465589739' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T20:29:26,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37083 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:59098 deadline: 1733776172608, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7227005361465589739' does not exist 2024-12-09T20:29:26,617 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 46f7c943b322,37083,1733776112533 for closing scanner id = -7227005361465589739 for 6e53f96f465f63d860346b347adf8ce9 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7227005361465589739' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:29:26,621 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=224 (was 222) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=159 (was 169), ProcessCount=11 (was 11), AvailableMemoryMB=5680 (was 5694) 2024-12-09T20:29:26,631 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=224, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=159, ProcessCount=11, AvailableMemoryMB=5677 2024-12-09T20:29:26,634 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:29:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-12-09T20:29:26,637 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T20:29:26,638 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:26,638 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-12-09T20:29:26,639 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T20:29:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T20:29:26,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741841_1017 (size=423) 2024-12-09T20:29:27,051 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 04a1637df756cbd7e50b91976c95b02c, NAME => 'testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05 2024-12-09T20:29:27,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741842_1018 (size=74) 2024-12-09T20:29:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-12-09T20:29:27,460 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:29:27,461 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 04a1637df756cbd7e50b91976c95b02c, disabling compactions & flushes 2024-12-09T20:29:27,461 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c. 2024-12-09T20:29:27,461 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c. 2024-12-09T20:29:27,461 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c. after waiting 0 ms 2024-12-09T20:29:27,461 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c. 2024-12-09T20:29:27,461 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c. 2024-12-09T20:29:27,461 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 04a1637df756cbd7e50b91976c95b02c: Waiting for close lock at 1733776167460Disabling compacts and flushes for region at 1733776167460Disabling writes for close at 1733776167461 (+1 ms)Writing region close event to WAL at 1733776167461Closed at 1733776167461 2024-12-09T20:29:27,463 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T20:29:27,463 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733776167463"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733776167463"}]},"ts":"1733776167463"} 2024-12-09T20:29:27,466 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T20:29:27,468 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T20:29:27,468 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776167468"}]},"ts":"1733776167468"} 2024-12-09T20:29:27,470 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-12-09T20:29:27,471 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=04a1637df756cbd7e50b91976c95b02c, ASSIGN}] 2024-12-09T20:29:27,473 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=04a1637df756cbd7e50b91976c95b02c, ASSIGN 2024-12-09T20:29:27,474 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=04a1637df756cbd7e50b91976c95b02c, ASSIGN; state=OFFLINE, location=46f7c943b322,37083,1733776112533; forceNewPlan=false, retain=false 2024-12-09T20:29:27,625 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=04a1637df756cbd7e50b91976c95b02c, regionState=OPENING, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:29:27,628 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=04a1637df756cbd7e50b91976c95b02c, ASSIGN because future has completed 2024-12-09T20:29:27,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04a1637df756cbd7e50b91976c95b02c, server=46f7c943b322,37083,1733776112533}] 2024-12-09T20:29:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T20:29:27,788 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c. 2024-12-09T20:29:27,788 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 04a1637df756cbd7e50b91976c95b02c, NAME => 'testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:29:27,789 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 04a1637df756cbd7e50b91976c95b02c 2024-12-09T20:29:27,789 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:29:27,789 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 04a1637df756cbd7e50b91976c95b02c 2024-12-09T20:29:27,789 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 04a1637df756cbd7e50b91976c95b02c 2024-12-09T20:29:27,792 INFO [StoreOpener-04a1637df756cbd7e50b91976c95b02c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 04a1637df756cbd7e50b91976c95b02c 2024-12-09T20:29:27,794 INFO [StoreOpener-04a1637df756cbd7e50b91976c95b02c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04a1637df756cbd7e50b91976c95b02c columnFamilyName testFamily 2024-12-09T20:29:27,795 DEBUG [StoreOpener-04a1637df756cbd7e50b91976c95b02c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:27,795 INFO [StoreOpener-04a1637df756cbd7e50b91976c95b02c-1 {}] regionserver.HStore(327): Store=04a1637df756cbd7e50b91976c95b02c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:29:27,796 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 04a1637df756cbd7e50b91976c95b02c 2024-12-09T20:29:27,797 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testRetryOutOfOrderScannerNextException/04a1637df756cbd7e50b91976c95b02c 2024-12-09T20:29:27,797 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testRetryOutOfOrderScannerNextException/04a1637df756cbd7e50b91976c95b02c 2024-12-09T20:29:27,798 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 04a1637df756cbd7e50b91976c95b02c 2024-12-09T20:29:27,798 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 04a1637df756cbd7e50b91976c95b02c 2024-12-09T20:29:27,801 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 04a1637df756cbd7e50b91976c95b02c 2024-12-09T20:29:27,805 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testRetryOutOfOrderScannerNextException/04a1637df756cbd7e50b91976c95b02c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T20:29:27,805 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 04a1637df756cbd7e50b91976c95b02c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66747024, jitterRate=-0.005391836166381836}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T20:29:27,806 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 04a1637df756cbd7e50b91976c95b02c 2024-12-09T20:29:27,806 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 04a1637df756cbd7e50b91976c95b02c: Running coprocessor pre-open hook at 1733776167790Writing region info on filesystem at 1733776167790Initializing all the Stores at 1733776167791 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776167791Cleaning up temporary data from old regions at 1733776167798 (+7 ms)Running coprocessor post-open hooks at 1733776167806 (+8 ms)Region opened successfully at 1733776167806 2024-12-09T20:29:27,808 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c., pid=15, masterSystemTime=1733776167782 2024-12-09T20:29:27,811 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c. 2024-12-09T20:29:27,811 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c. 2024-12-09T20:29:27,812 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=04a1637df756cbd7e50b91976c95b02c, regionState=OPEN, openSeqNum=2, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:29:27,815 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04a1637df756cbd7e50b91976c95b02c, server=46f7c943b322,37083,1733776112533 because future has completed 2024-12-09T20:29:27,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-09T20:29:27,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 04a1637df756cbd7e50b91976c95b02c, server=46f7c943b322,37083,1733776112533 in 188 msec 2024-12-09T20:29:27,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-09T20:29:27,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=04a1637df756cbd7e50b91976c95b02c, ASSIGN in 349 msec 2024-12-09T20:29:27,825 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T20:29:27,825 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776167825"}]},"ts":"1733776167825"} 2024-12-09T20:29:27,827 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-12-09T20:29:27,829 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T20:29:27,832 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.1950 sec 2024-12-09T20:29:28,383 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-12-09T20:29:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T20:29:29,661 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-12-09T20:29:29,662 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-12-09T20:29:29,662 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T20:29:29,668 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-12-09T20:29:29,668 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T20:29:29,669 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-12-09T20:29:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c., hostname=46f7c943b322,37083,1733776112533, seqNum=2] 2024-12-09T20:29:29,686 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T20:29:29,686 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-12-09T20:29:29,691 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-09T20:29:29,693 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-09T20:29:29,695 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-09T20:29:29,696 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-7227005361465589735 2024-12-09T20:29:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:59098 deadline: 1733776172695, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T20:29:29,699 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-09T20:29:29,701 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-09T20:29:29,702 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T20:29:29,704 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T20:29:29,704 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-7227005361465589733 2024-12-09T20:29:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:59098 deadline: 1733776172703, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T20:29:29,708 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T20:29:29,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-7227005361465589732 2024-12-09T20:29:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:59098 deadline: 1733776172708, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T20:29:29,713 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T20:29:29,714 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-7227005361465589731 2024-12-09T20:29:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:59098 deadline: 1733776172712, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-09T20:29:29,718 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-09T20:29:29,727 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=227 (was 224) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-543335734_22 at /127.0.0.1:57048 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=154 (was 159), ProcessCount=11 (was 11), AvailableMemoryMB=5661 (was 5677) 2024-12-09T20:29:29,737 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=227, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=154, ProcessCount=11, AvailableMemoryMB=5656 2024-12-09T20:29:29,741 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:29:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-12-09T20:29:29,745 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T20:29:29,746 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:29,746 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-12-09T20:29:29,747 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T20:29:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-09T20:29:29,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741843_1019 (size=418) 2024-12-09T20:29:30,162 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 993e0dbbafe0192b19ca29ebc1a69fd1, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05 2024-12-09T20:29:30,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741844_1020 (size=69) 2024-12-09T20:29:30,572 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:29:30,573 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 993e0dbbafe0192b19ca29ebc1a69fd1, disabling compactions & flushes 2024-12-09T20:29:30,573 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1. 2024-12-09T20:29:30,573 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1. 2024-12-09T20:29:30,573 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1. after waiting 0 ms 2024-12-09T20:29:30,573 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1. 2024-12-09T20:29:30,573 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1. 2024-12-09T20:29:30,573 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 993e0dbbafe0192b19ca29ebc1a69fd1: Waiting for close lock at 1733776170573Disabling compacts and flushes for region at 1733776170573Disabling writes for close at 1733776170573Writing region close event to WAL at 1733776170573Closed at 1733776170573 2024-12-09T20:29:30,575 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T20:29:30,575 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1733776170575"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733776170575"}]},"ts":"1733776170575"} 2024-12-09T20:29:30,579 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T20:29:30,580 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T20:29:30,580 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776170580"}]},"ts":"1733776170580"} 2024-12-09T20:29:30,583 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-09T20:29:30,583 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=993e0dbbafe0192b19ca29ebc1a69fd1, ASSIGN}] 2024-12-09T20:29:30,585 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=993e0dbbafe0192b19ca29ebc1a69fd1, ASSIGN 2024-12-09T20:29:30,586 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=993e0dbbafe0192b19ca29ebc1a69fd1, ASSIGN; state=OFFLINE, location=46f7c943b322,37083,1733776112533; forceNewPlan=false, retain=false 2024-12-09T20:29:30,718 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T20:29:30,737 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=993e0dbbafe0192b19ca29ebc1a69fd1, regionState=OPENING, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:29:30,741 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=993e0dbbafe0192b19ca29ebc1a69fd1, ASSIGN because future has completed 2024-12-09T20:29:30,741 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 993e0dbbafe0192b19ca29ebc1a69fd1, server=46f7c943b322,37083,1733776112533}] 2024-12-09T20:29:30,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-09T20:29:30,905 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1. 2024-12-09T20:29:30,905 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 993e0dbbafe0192b19ca29ebc1a69fd1, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:29:30,905 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 993e0dbbafe0192b19ca29ebc1a69fd1 2024-12-09T20:29:30,905 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:29:30,906 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 993e0dbbafe0192b19ca29ebc1a69fd1 2024-12-09T20:29:30,906 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 993e0dbbafe0192b19ca29ebc1a69fd1 2024-12-09T20:29:30,908 INFO [StoreOpener-993e0dbbafe0192b19ca29ebc1a69fd1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 993e0dbbafe0192b19ca29ebc1a69fd1 2024-12-09T20:29:30,910 INFO [StoreOpener-993e0dbbafe0192b19ca29ebc1a69fd1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 993e0dbbafe0192b19ca29ebc1a69fd1 columnFamilyName testFamily 2024-12-09T20:29:30,910 DEBUG [StoreOpener-993e0dbbafe0192b19ca29ebc1a69fd1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:30,911 INFO [StoreOpener-993e0dbbafe0192b19ca29ebc1a69fd1-1 {}] regionserver.HStore(327): Store=993e0dbbafe0192b19ca29ebc1a69fd1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:29:30,911 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 993e0dbbafe0192b19ca29ebc1a69fd1 2024-12-09T20:29:30,912 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnOpenScanner/993e0dbbafe0192b19ca29ebc1a69fd1 2024-12-09T20:29:30,913 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnOpenScanner/993e0dbbafe0192b19ca29ebc1a69fd1 2024-12-09T20:29:30,913 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 993e0dbbafe0192b19ca29ebc1a69fd1 2024-12-09T20:29:30,913 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 993e0dbbafe0192b19ca29ebc1a69fd1 2024-12-09T20:29:30,916 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 993e0dbbafe0192b19ca29ebc1a69fd1 2024-12-09T20:29:30,919 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnOpenScanner/993e0dbbafe0192b19ca29ebc1a69fd1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T20:29:30,919 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 993e0dbbafe0192b19ca29ebc1a69fd1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74891887, jitterRate=0.11597608029842377}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T20:29:30,920 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 993e0dbbafe0192b19ca29ebc1a69fd1 2024-12-09T20:29:30,920 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 993e0dbbafe0192b19ca29ebc1a69fd1: Running coprocessor pre-open hook at 1733776170906Writing region info on filesystem at 1733776170906Initializing all the Stores at 1733776170907 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776170907Cleaning up temporary data from old regions at 1733776170913 (+6 ms)Running coprocessor post-open hooks at 1733776170920 (+7 ms)Region opened successfully at 1733776170920 2024-12-09T20:29:30,922 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1., pid=18, masterSystemTime=1733776170894 2024-12-09T20:29:30,925 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1. 2024-12-09T20:29:30,925 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1. 2024-12-09T20:29:30,925 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=993e0dbbafe0192b19ca29ebc1a69fd1, regionState=OPEN, openSeqNum=2, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:29:30,929 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 993e0dbbafe0192b19ca29ebc1a69fd1, server=46f7c943b322,37083,1733776112533 because future has completed 2024-12-09T20:29:30,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-09T20:29:30,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 993e0dbbafe0192b19ca29ebc1a69fd1, server=46f7c943b322,37083,1733776112533 in 189 msec 2024-12-09T20:29:30,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-09T20:29:30,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=993e0dbbafe0192b19ca29ebc1a69fd1, ASSIGN in 350 msec 2024-12-09T20:29:30,937 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T20:29:30,938 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776170938"}]},"ts":"1733776170938"} 2024-12-09T20:29:30,940 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-09T20:29:30,941 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T20:29:30,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2000 sec 2024-12-09T20:29:32,243 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-09T20:29:32,243 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-12-09T20:29:32,244 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-09T20:29:32,244 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-09T20:29:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-09T20:29:32,771 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-12-09T20:29:32,771 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-09T20:29:32,772 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T20:29:32,777 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-09T20:29:32,777 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T20:29:32,777 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-12-09T20:29:32,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1., hostname=46f7c943b322,37083,1733776112533, seqNum=2] 2024-12-09T20:29:32,792 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T20:29:32,792 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-09T20:29:32,795 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-09T20:29:32,797 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-09T20:29:33,291 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T20:29:33,797 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1., hostname=46f7c943b322,37083,1733776112533, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1., hostname=46f7c943b322,37083,1733776112533, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=46f7c943b322:37083 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1001ms, rpcTimeout=1000ms 2024-12-09T20:29:33,797 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1., hostname=46f7c943b322,37083,1733776112533, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1001ms, rpcTimeout=1000ms 2024-12-09T20:29:33,797 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1., hostname=46f7c943b322,37083,1733776112533, seqNum=2 from cache 2024-12-09T20:29:34,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-12-09T20:29:34,384 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-12-09T20:29:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1., hostname=46f7c943b322,37083,1733776112533, seqNum=2] 2024-12-09T20:29:34,814 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-09T20:29:35,797 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1., hostname=46f7c943b322,37083,1733776112533, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1., hostname=46f7c943b322,37083,1733776112533, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=46f7c943b322:37083 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=984ms, rpcTimeout=982ms 2024-12-09T20:29:35,798 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1., hostname=46f7c943b322,37083,1733776112533, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=984ms, rpcTimeout=982ms 2024-12-09T20:29:35,798 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1., hostname=46f7c943b322,37083,1733776112533, seqNum=2 from cache 2024-12-09T20:29:35,807 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 227), OpenFileDescriptor=448 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=149 (was 154), ProcessCount=11 (was 11), AvailableMemoryMB=5678 (was 5656) - AvailableMemoryMB LEAK? - 2024-12-09T20:29:35,815 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=149, ProcessCount=11, AvailableMemoryMB=5668 2024-12-09T20:29:35,818 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:29:35,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-12-09T20:29:35,822 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T20:29:35,822 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:35,822 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-12-09T20:29:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T20:29:35,823 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T20:29:35,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741845_1021 (size=416) 2024-12-09T20:29:36,235 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8ed7d59ddc232f701aeec84df76a6165, NAME => 'testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05 2024-12-09T20:29:36,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741846_1022 (size=67) 2024-12-09T20:29:36,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-12-09T20:29:36,644 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:29:36,644 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 8ed7d59ddc232f701aeec84df76a6165, disabling compactions & flushes 2024-12-09T20:29:36,644 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165. 2024-12-09T20:29:36,644 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165. 2024-12-09T20:29:36,644 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165. after waiting 0 ms 2024-12-09T20:29:36,644 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165. 2024-12-09T20:29:36,644 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165. 2024-12-09T20:29:36,645 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8ed7d59ddc232f701aeec84df76a6165: Waiting for close lock at 1733776176644Disabling compacts and flushes for region at 1733776176644Disabling writes for close at 1733776176644Writing region close event to WAL at 1733776176644Closed at 1733776176644 2024-12-09T20:29:36,647 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T20:29:36,647 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1733776176647"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733776176647"}]},"ts":"1733776176647"} 2024-12-09T20:29:36,650 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T20:29:36,652 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T20:29:36,652 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776176652"}]},"ts":"1733776176652"} 2024-12-09T20:29:36,654 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-09T20:29:36,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=8ed7d59ddc232f701aeec84df76a6165, ASSIGN}] 2024-12-09T20:29:36,656 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=8ed7d59ddc232f701aeec84df76a6165, ASSIGN 2024-12-09T20:29:36,657 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=8ed7d59ddc232f701aeec84df76a6165, ASSIGN; state=OFFLINE, location=46f7c943b322,37083,1733776112533; forceNewPlan=false, retain=false 2024-12-09T20:29:36,808 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=8ed7d59ddc232f701aeec84df76a6165, regionState=OPENING, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:29:36,811 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=8ed7d59ddc232f701aeec84df76a6165, ASSIGN because future has completed 2024-12-09T20:29:36,812 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ed7d59ddc232f701aeec84df76a6165, server=46f7c943b322,37083,1733776112533}] 2024-12-09T20:29:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T20:29:36,971 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165. 2024-12-09T20:29:36,971 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ed7d59ddc232f701aeec84df76a6165, NAME => 'testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:29:36,972 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 8ed7d59ddc232f701aeec84df76a6165 2024-12-09T20:29:36,972 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:29:36,972 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 8ed7d59ddc232f701aeec84df76a6165 2024-12-09T20:29:36,972 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 8ed7d59ddc232f701aeec84df76a6165 2024-12-09T20:29:36,974 INFO [StoreOpener-8ed7d59ddc232f701aeec84df76a6165-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8ed7d59ddc232f701aeec84df76a6165 2024-12-09T20:29:36,977 INFO [StoreOpener-8ed7d59ddc232f701aeec84df76a6165-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ed7d59ddc232f701aeec84df76a6165 columnFamilyName testFamily 2024-12-09T20:29:36,977 DEBUG [StoreOpener-8ed7d59ddc232f701aeec84df76a6165-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:36,977 INFO [StoreOpener-8ed7d59ddc232f701aeec84df76a6165-1 {}] regionserver.HStore(327): Store=8ed7d59ddc232f701aeec84df76a6165/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:29:36,978 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 8ed7d59ddc232f701aeec84df76a6165 2024-12-09T20:29:36,979 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnNextAsync/8ed7d59ddc232f701aeec84df76a6165 2024-12-09T20:29:36,979 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnNextAsync/8ed7d59ddc232f701aeec84df76a6165 2024-12-09T20:29:36,980 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 8ed7d59ddc232f701aeec84df76a6165 2024-12-09T20:29:36,980 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 8ed7d59ddc232f701aeec84df76a6165 2024-12-09T20:29:36,982 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 8ed7d59ddc232f701aeec84df76a6165 2024-12-09T20:29:36,985 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnNextAsync/8ed7d59ddc232f701aeec84df76a6165/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T20:29:36,986 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 8ed7d59ddc232f701aeec84df76a6165; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74183051, jitterRate=0.10541360080242157}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T20:29:36,986 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ed7d59ddc232f701aeec84df76a6165 2024-12-09T20:29:36,986 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 8ed7d59ddc232f701aeec84df76a6165: Running coprocessor pre-open hook at 1733776176972Writing region info on filesystem at 1733776176972Initializing all the Stores at 1733776176974 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776176974Cleaning up temporary data from old regions at 1733776176980 (+6 ms)Running coprocessor post-open hooks at 1733776176986 (+6 ms)Region opened successfully at 1733776176986 2024-12-09T20:29:36,988 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165., pid=21, masterSystemTime=1733776176965 2024-12-09T20:29:36,991 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165. 2024-12-09T20:29:36,991 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165. 2024-12-09T20:29:36,992 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=8ed7d59ddc232f701aeec84df76a6165, regionState=OPEN, openSeqNum=2, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:29:36,995 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ed7d59ddc232f701aeec84df76a6165, server=46f7c943b322,37083,1733776112533 because future has completed 2024-12-09T20:29:36,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-09T20:29:36,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ed7d59ddc232f701aeec84df76a6165, server=46f7c943b322,37083,1733776112533 in 185 msec 2024-12-09T20:29:37,001 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-09T20:29:37,002 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=8ed7d59ddc232f701aeec84df76a6165, ASSIGN in 345 msec 2024-12-09T20:29:37,002 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T20:29:37,003 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776177002"}]},"ts":"1733776177002"} 2024-12-09T20:29:37,005 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-09T20:29:37,006 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T20:29:37,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.1880 sec 2024-12-09T20:29:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T20:29:38,852 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-12-09T20:29:38,852 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-09T20:29:38,852 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T20:29:38,858 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-09T20:29:38,858 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T20:29:38,858 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-12-09T20:29:38,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165., hostname=46f7c943b322,37083,1733776112533, seqNum=2] 2024-12-09T20:29:38,873 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T20:29:38,873 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-09T20:29:38,876 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-09T20:29:38,878 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-09T20:29:38,879 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-09T20:29:38,880 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-09T20:29:40,385 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-12-09T20:29:41,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37083 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7227005361465589720' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T20:29:41,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37083 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:59098 deadline: 1733776182889, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7227005361465589720' does not exist 2024-12-09T20:29:41,891 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 46f7c943b322,37083,1733776112533 for closing scanner id = -7227005361465589720 for 8ed7d59ddc232f701aeec84df76a6165 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7227005361465589720' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:29:41,896 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=145 (was 149), ProcessCount=11 (was 11), AvailableMemoryMB=5675 (was 5668) - AvailableMemoryMB LEAK? - 2024-12-09T20:29:41,904 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=145, ProcessCount=11, AvailableMemoryMB=5675 2024-12-09T20:29:41,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:29:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-12-09T20:29:41,909 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T20:29:41,909 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:41,910 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-12-09T20:29:41,910 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T20:29:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-09T20:29:41,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741847_1023 (size=411) 2024-12-09T20:29:42,321 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d6f662cbb15d92024c09dcbb908e62ba, NAME => 'testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05 2024-12-09T20:29:42,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741848_1024 (size=62) 2024-12-09T20:29:42,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-12-09T20:29:42,473 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T20:29:42,504 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-09T20:29:42,504 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-09T20:29:42,729 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:29:42,730 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing d6f662cbb15d92024c09dcbb908e62ba, disabling compactions & flushes 2024-12-09T20:29:42,730 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba. 2024-12-09T20:29:42,730 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba. 2024-12-09T20:29:42,730 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba. after waiting 0 ms 2024-12-09T20:29:42,730 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba. 2024-12-09T20:29:42,730 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba. 2024-12-09T20:29:42,730 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for d6f662cbb15d92024c09dcbb908e62ba: Waiting for close lock at 1733776182730Disabling compacts and flushes for region at 1733776182730Disabling writes for close at 1733776182730Writing region close event to WAL at 1733776182730Closed at 1733776182730 2024-12-09T20:29:42,732 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T20:29:42,732 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1733776182732"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733776182732"}]},"ts":"1733776182732"} 2024-12-09T20:29:42,735 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T20:29:42,736 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T20:29:42,737 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776182736"}]},"ts":"1733776182736"} 2024-12-09T20:29:42,739 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-09T20:29:42,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=d6f662cbb15d92024c09dcbb908e62ba, ASSIGN}] 2024-12-09T20:29:42,741 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=d6f662cbb15d92024c09dcbb908e62ba, ASSIGN 2024-12-09T20:29:42,742 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=d6f662cbb15d92024c09dcbb908e62ba, ASSIGN; state=OFFLINE, location=46f7c943b322,37083,1733776112533; forceNewPlan=false, retain=false 2024-12-09T20:29:42,893 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=d6f662cbb15d92024c09dcbb908e62ba, regionState=OPENING, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:29:42,896 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=d6f662cbb15d92024c09dcbb908e62ba, ASSIGN because future has completed 2024-12-09T20:29:42,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6f662cbb15d92024c09dcbb908e62ba, server=46f7c943b322,37083,1733776112533}] 2024-12-09T20:29:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-09T20:29:43,055 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba. 2024-12-09T20:29:43,055 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => d6f662cbb15d92024c09dcbb908e62ba, NAME => 'testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:29:43,055 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:29:43,056 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:29:43,056 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:29:43,056 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:29:43,058 INFO [StoreOpener-d6f662cbb15d92024c09dcbb908e62ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:29:43,060 INFO [StoreOpener-d6f662cbb15d92024c09dcbb908e62ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6f662cbb15d92024c09dcbb908e62ba columnFamilyName testFamily 2024-12-09T20:29:43,060 DEBUG [StoreOpener-d6f662cbb15d92024c09dcbb908e62ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:43,061 INFO [StoreOpener-d6f662cbb15d92024c09dcbb908e62ba-1 {}] regionserver.HStore(327): Store=d6f662cbb15d92024c09dcbb908e62ba/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:29:43,061 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:29:43,062 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnNext/d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:29:43,062 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnNext/d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:29:43,063 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:29:43,063 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:29:43,065 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:29:43,068 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnNext/d6f662cbb15d92024c09dcbb908e62ba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T20:29:43,069 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened d6f662cbb15d92024c09dcbb908e62ba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69606695, jitterRate=0.037220582365989685}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T20:29:43,069 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:29:43,070 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for d6f662cbb15d92024c09dcbb908e62ba: Running coprocessor pre-open hook at 1733776183056Writing region info on filesystem at 1733776183056Initializing all the Stores at 1733776183057 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776183057Cleaning up temporary data from old regions at 1733776183063 (+6 ms)Running coprocessor post-open hooks at 1733776183069 (+6 ms)Region opened successfully at 1733776183069 2024-12-09T20:29:43,071 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba., pid=24, masterSystemTime=1733776183049 2024-12-09T20:29:43,074 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba. 2024-12-09T20:29:43,074 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba. 2024-12-09T20:29:43,075 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=d6f662cbb15d92024c09dcbb908e62ba, regionState=OPEN, openSeqNum=2, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:29:43,078 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure d6f662cbb15d92024c09dcbb908e62ba, server=46f7c943b322,37083,1733776112533 because future has completed 2024-12-09T20:29:43,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-09T20:29:43,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure d6f662cbb15d92024c09dcbb908e62ba, server=46f7c943b322,37083,1733776112533 in 183 msec 2024-12-09T20:29:43,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-09T20:29:43,085 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=d6f662cbb15d92024c09dcbb908e62ba, ASSIGN in 343 msec 2024-12-09T20:29:43,086 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T20:29:43,086 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776183086"}]},"ts":"1733776183086"} 2024-12-09T20:29:43,089 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-09T20:29:43,090 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T20:29:43,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.1840 sec 2024-12-09T20:29:44,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-09T20:29:44,942 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-12-09T20:29:44,942 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-09T20:29:44,943 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T20:29:44,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-09T20:29:44,948 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T20:29:44,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-12-09T20:29:44,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba., hostname=46f7c943b322,37083,1733776112533, seqNum=2] 2024-12-09T20:29:44,961 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T20:29:44,961 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-09T20:29:44,964 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-09T20:29:44,966 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-09T20:29:44,967 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-09T20:29:44,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-09T20:29:45,088 INFO [master/46f7c943b322:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-09T20:29:45,088 INFO [master/46f7c943b322:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-09T20:29:46,386 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-12-09T20:29:47,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37083 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7227005361465589716' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T20:29:47,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37083 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:59098 deadline: 1733776188970, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7227005361465589716' does not exist 2024-12-09T20:29:47,971 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 46f7c943b322,37083,1733776112533 for closing scanner id = -7227005361465589716 for d6f662cbb15d92024c09dcbb908e62ba of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7227005361465589716' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:29:47,977 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=142 (was 145), ProcessCount=11 (was 11), AvailableMemoryMB=5617 (was 5675) 2024-12-09T20:29:47,988 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=142, ProcessCount=11, AvailableMemoryMB=5616 2024-12-09T20:29:47,991 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36137 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:29:47,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36137 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-12-09T20:29:47,994 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T20:29:47,994 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:47,994 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36137 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-12-09T20:29:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T20:29:47,995 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T20:29:48,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741849_1025 (size=419) 2024-12-09T20:29:48,406 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d44012135436c54459a77d97b6eda809, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05 2024-12-09T20:29:48,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741850_1026 (size=70) 2024-12-09T20:29:48,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-12-09T20:29:48,557 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T20:29:48,815 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:29:48,815 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing d44012135436c54459a77d97b6eda809, disabling compactions & flushes 2024-12-09T20:29:48,815 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809. 2024-12-09T20:29:48,816 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809. 2024-12-09T20:29:48,816 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809. after waiting 0 ms 2024-12-09T20:29:48,816 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809. 2024-12-09T20:29:48,816 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809. 2024-12-09T20:29:48,816 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for d44012135436c54459a77d97b6eda809: Waiting for close lock at 1733776188815Disabling compacts and flushes for region at 1733776188815Disabling writes for close at 1733776188816 (+1 ms)Writing region close event to WAL at 1733776188816Closed at 1733776188816 2024-12-09T20:29:48,818 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T20:29:48,818 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1733776188818"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733776188818"}]},"ts":"1733776188818"} 2024-12-09T20:29:48,821 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T20:29:48,822 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T20:29:48,822 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776188822"}]},"ts":"1733776188822"} 2024-12-09T20:29:48,825 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-09T20:29:48,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=d44012135436c54459a77d97b6eda809, ASSIGN}] 2024-12-09T20:29:48,826 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=d44012135436c54459a77d97b6eda809, ASSIGN 2024-12-09T20:29:48,827 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=d44012135436c54459a77d97b6eda809, ASSIGN; state=OFFLINE, location=46f7c943b322,37083,1733776112533; forceNewPlan=false, retain=false 2024-12-09T20:29:48,978 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=d44012135436c54459a77d97b6eda809, regionState=OPENING, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:29:48,982 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=d44012135436c54459a77d97b6eda809, ASSIGN because future has completed 2024-12-09T20:29:48,982 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure d44012135436c54459a77d97b6eda809, server=46f7c943b322,37083,1733776112533}] 2024-12-09T20:29:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T20:29:49,140 INFO [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809. 2024-12-09T20:29:49,140 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => d44012135436c54459a77d97b6eda809, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:29:49,141 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync d44012135436c54459a77d97b6eda809 2024-12-09T20:29:49,141 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:29:49,141 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for d44012135436c54459a77d97b6eda809 2024-12-09T20:29:49,141 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for d44012135436c54459a77d97b6eda809 2024-12-09T20:29:49,143 INFO [StoreOpener-d44012135436c54459a77d97b6eda809-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d44012135436c54459a77d97b6eda809 2024-12-09T20:29:49,145 INFO [StoreOpener-d44012135436c54459a77d97b6eda809-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d44012135436c54459a77d97b6eda809 columnFamilyName testFamily 2024-12-09T20:29:49,145 DEBUG [StoreOpener-d44012135436c54459a77d97b6eda809-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:49,145 INFO [StoreOpener-d44012135436c54459a77d97b6eda809-1 {}] regionserver.HStore(327): Store=d44012135436c54459a77d97b6eda809/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:29:49,146 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for d44012135436c54459a77d97b6eda809 2024-12-09T20:29:49,147 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d44012135436c54459a77d97b6eda809 2024-12-09T20:29:49,147 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d44012135436c54459a77d97b6eda809 2024-12-09T20:29:49,148 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for d44012135436c54459a77d97b6eda809 2024-12-09T20:29:49,148 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for d44012135436c54459a77d97b6eda809 2024-12-09T20:29:49,151 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for d44012135436c54459a77d97b6eda809 2024-12-09T20:29:49,154 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d44012135436c54459a77d97b6eda809/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T20:29:49,155 INFO [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened d44012135436c54459a77d97b6eda809; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72515388, jitterRate=0.080563485622406}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T20:29:49,155 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d44012135436c54459a77d97b6eda809 2024-12-09T20:29:49,156 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for d44012135436c54459a77d97b6eda809: Running coprocessor pre-open hook at 1733776189141Writing region info on filesystem at 1733776189141Initializing all the Stores at 1733776189142 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776189142Cleaning up temporary data from old regions at 1733776189148 (+6 ms)Running coprocessor post-open hooks at 1733776189155 (+7 ms)Region opened successfully at 1733776189156 (+1 ms) 2024-12-09T20:29:49,157 INFO [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809., pid=27, masterSystemTime=1733776189134 2024-12-09T20:29:49,160 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809. 2024-12-09T20:29:49,160 INFO [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809. 2024-12-09T20:29:49,161 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=d44012135436c54459a77d97b6eda809, regionState=OPEN, openSeqNum=2, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:29:49,163 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure d44012135436c54459a77d97b6eda809, server=46f7c943b322,37083,1733776112533 because future has completed 2024-12-09T20:29:49,167 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-09T20:29:49,167 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure d44012135436c54459a77d97b6eda809, server=46f7c943b322,37083,1733776112533 in 182 msec 2024-12-09T20:29:49,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-09T20:29:49,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=d44012135436c54459a77d97b6eda809, ASSIGN in 342 msec 2024-12-09T20:29:49,170 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T20:29:49,171 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776189170"}]},"ts":"1733776189170"} 2024-12-09T20:29:49,173 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-09T20:29:49,174 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T20:29:49,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.1830 sec 2024-12-09T20:29:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T20:29:51,021 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-12-09T20:29:51,021 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-09T20:29:51,022 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T20:29:51,026 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-09T20:29:51,026 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T20:29:51,027 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-12-09T20:29:51,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809., hostname=46f7c943b322,37083,1733776112533, seqNum=2] 2024-12-09T20:29:51,039 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T20:29:51,039 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-09T20:29:51,042 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-09T20:29:51,043 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37083 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-09T20:29:52,243 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-09T20:29:52,243 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-09T20:29:52,244 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-09T20:29:52,244 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-09T20:29:52,388 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-12-09T20:29:54,643 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T20:29:57,047 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809., hostname=46f7c943b322,37083,1733776112533, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809., hostname=46f7c943b322,37083,1733776112533, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=46f7c943b322:37083 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6005ms, rpcTimeout=5999ms 2024-12-09T20:29:57,047 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809., hostname=46f7c943b322,37083,1733776112533, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6005ms, rpcTimeout=5999ms 2024-12-09T20:29:57,047 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809., hostname=46f7c943b322,37083,1733776112533, seqNum=2 from cache 2024-12-09T20:29:57,057 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 142), ProcessCount=11 (was 11), AvailableMemoryMB=5666 (was 5616) - AvailableMemoryMB LEAK? - 2024-12-09T20:29:57,065 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=5666 2024-12-09T20:29:57,067 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36137 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:29:57,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36137 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-12-09T20:29:57,070 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T20:29:57,070 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:57,070 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36137 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-12-09T20:29:57,071 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T20:29:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-09T20:29:57,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741851_1027 (size=407) 2024-12-09T20:29:57,482 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 977459cd283f0fa2ff2dd4b39ab43f0a, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05 2024-12-09T20:29:57,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741852_1028 (size=58) 2024-12-09T20:29:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-12-09T20:29:57,892 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:29:57,892 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 977459cd283f0fa2ff2dd4b39ab43f0a, disabling compactions & flushes 2024-12-09T20:29:57,892 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a. 2024-12-09T20:29:57,892 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a. 2024-12-09T20:29:57,892 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a. after waiting 0 ms 2024-12-09T20:29:57,892 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a. 2024-12-09T20:29:57,892 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a. 2024-12-09T20:29:57,892 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 977459cd283f0fa2ff2dd4b39ab43f0a: Waiting for close lock at 1733776197892Disabling compacts and flushes for region at 1733776197892Disabling writes for close at 1733776197892Writing region close event to WAL at 1733776197892Closed at 1733776197892 2024-12-09T20:29:57,894 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T20:29:57,894 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1733776197894"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733776197894"}]},"ts":"1733776197894"} 2024-12-09T20:29:57,896 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T20:29:57,898 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T20:29:57,898 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776197898"}]},"ts":"1733776197898"} 2024-12-09T20:29:57,900 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-09T20:29:57,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=977459cd283f0fa2ff2dd4b39ab43f0a, ASSIGN}] 2024-12-09T20:29:57,901 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=977459cd283f0fa2ff2dd4b39ab43f0a, ASSIGN 2024-12-09T20:29:57,903 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=977459cd283f0fa2ff2dd4b39ab43f0a, ASSIGN; state=OFFLINE, location=46f7c943b322,37083,1733776112533; forceNewPlan=false, retain=false 2024-12-09T20:29:58,053 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=977459cd283f0fa2ff2dd4b39ab43f0a, regionState=OPENING, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:29:58,056 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=977459cd283f0fa2ff2dd4b39ab43f0a, ASSIGN because future has completed 2024-12-09T20:29:58,057 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 977459cd283f0fa2ff2dd4b39ab43f0a, server=46f7c943b322,37083,1733776112533}] 2024-12-09T20:29:58,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-09T20:29:58,215 INFO [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a. 2024-12-09T20:29:58,215 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 977459cd283f0fa2ff2dd4b39ab43f0a, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:29:58,216 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 977459cd283f0fa2ff2dd4b39ab43f0a 2024-12-09T20:29:58,216 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:29:58,216 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 977459cd283f0fa2ff2dd4b39ab43f0a 2024-12-09T20:29:58,216 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 977459cd283f0fa2ff2dd4b39ab43f0a 2024-12-09T20:29:58,218 INFO [StoreOpener-977459cd283f0fa2ff2dd4b39ab43f0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 977459cd283f0fa2ff2dd4b39ab43f0a 2024-12-09T20:29:58,219 INFO [StoreOpener-977459cd283f0fa2ff2dd4b39ab43f0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 977459cd283f0fa2ff2dd4b39ab43f0a columnFamilyName testFamily 2024-12-09T20:29:58,219 DEBUG [StoreOpener-977459cd283f0fa2ff2dd4b39ab43f0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:29:58,220 INFO [StoreOpener-977459cd283f0fa2ff2dd4b39ab43f0a-1 {}] regionserver.HStore(327): Store=977459cd283f0fa2ff2dd4b39ab43f0a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:29:58,220 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 977459cd283f0fa2ff2dd4b39ab43f0a 2024-12-09T20:29:58,221 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnNext/977459cd283f0fa2ff2dd4b39ab43f0a 2024-12-09T20:29:58,221 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnNext/977459cd283f0fa2ff2dd4b39ab43f0a 2024-12-09T20:29:58,223 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 977459cd283f0fa2ff2dd4b39ab43f0a 2024-12-09T20:29:58,223 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 977459cd283f0fa2ff2dd4b39ab43f0a 2024-12-09T20:29:58,224 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 977459cd283f0fa2ff2dd4b39ab43f0a 2024-12-09T20:29:58,227 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnNext/977459cd283f0fa2ff2dd4b39ab43f0a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T20:29:58,228 INFO [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 977459cd283f0fa2ff2dd4b39ab43f0a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73647646, jitterRate=0.09743544459342957}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T20:29:58,228 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 977459cd283f0fa2ff2dd4b39ab43f0a 2024-12-09T20:29:58,229 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 977459cd283f0fa2ff2dd4b39ab43f0a: Running coprocessor pre-open hook at 1733776198216Writing region info on filesystem at 1733776198216Initializing all the Stores at 1733776198217 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776198217Cleaning up temporary data from old regions at 1733776198223 (+6 ms)Running coprocessor post-open hooks at 1733776198228 (+5 ms)Region opened successfully at 1733776198229 (+1 ms) 2024-12-09T20:29:58,230 INFO [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a., pid=30, masterSystemTime=1733776198209 2024-12-09T20:29:58,232 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a. 2024-12-09T20:29:58,232 INFO [RS_OPEN_PRIORITY_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a. 2024-12-09T20:29:58,233 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=977459cd283f0fa2ff2dd4b39ab43f0a, regionState=OPEN, openSeqNum=2, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:29:58,236 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 977459cd283f0fa2ff2dd4b39ab43f0a, server=46f7c943b322,37083,1733776112533 because future has completed 2024-12-09T20:29:58,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-09T20:29:58,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 977459cd283f0fa2ff2dd4b39ab43f0a, server=46f7c943b322,37083,1733776112533 in 180 msec 2024-12-09T20:29:58,242 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-09T20:29:58,242 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=977459cd283f0fa2ff2dd4b39ab43f0a, ASSIGN in 339 msec 2024-12-09T20:29:58,242 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T20:29:58,243 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776198243"}]},"ts":"1733776198243"} 2024-12-09T20:29:58,245 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-09T20:29:58,245 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T20:29:58,247 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.1780 sec 2024-12-09T20:29:58,389 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-12-09T20:30:00,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-09T20:30:00,112 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-12-09T20:30:00,112 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-09T20:30:00,112 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T20:30:00,116 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-09T20:30:00,117 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T20:30:00,117 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-12-09T20:30:00,119 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a., hostname=46f7c943b322,37083,1733776112533, seqNum=2] 2024-12-09T20:30:00,128 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T20:30:00,128 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-09T20:30:00,131 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-09T20:30:00,132 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-09T20:30:00,133 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-09T20:30:00,134 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37083 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-09T20:30:00,718 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T20:30:02,243 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-09T20:30:02,243 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-09T20:30:03,718 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T20:30:05,171 DEBUG [master/46f7c943b322:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-09T20:30:06,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37083 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7227005361465589708' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-09T20:30:06,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37083 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:59098 deadline: 1733776212138, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7227005361465589708' does not exist 2024-12-09T20:30:06,141 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 46f7c943b322,37083,1733776112533 for closing scanner id = -7227005361465589708 for 977459cd283f0fa2ff2dd4b39ab43f0a of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-7227005361465589708' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T20:30:06,147 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=116 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=5597 (was 5666) 2024-12-09T20:30:06,156 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=116, ProcessCount=11, AvailableMemoryMB=5596 2024-12-09T20:30:06,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T20:30:06,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-12-09T20:30:06,162 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T20:30:06,162 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:30:06,162 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-12-09T20:30:06,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-09T20:30:06,163 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T20:30:06,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741853_1029 (size=423) 2024-12-09T20:30:06,575 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6c0df67a61025f1e2ea77a31af80ee4e, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05 2024-12-09T20:30:06,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741854_1030 (size=74) 2024-12-09T20:30:06,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-12-09T20:30:06,984 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:30:06,984 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 6c0df67a61025f1e2ea77a31af80ee4e, disabling compactions & flushes 2024-12-09T20:30:06,984 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e. 2024-12-09T20:30:06,984 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e. 2024-12-09T20:30:06,984 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e. after waiting 0 ms 2024-12-09T20:30:06,984 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e. 2024-12-09T20:30:06,984 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e. 2024-12-09T20:30:06,985 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6c0df67a61025f1e2ea77a31af80ee4e: Waiting for close lock at 1733776206984Disabling compacts and flushes for region at 1733776206984Disabling writes for close at 1733776206984Writing region close event to WAL at 1733776206984Closed at 1733776206984 2024-12-09T20:30:06,987 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T20:30:06,987 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733776206987"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733776206987"}]},"ts":"1733776206987"} 2024-12-09T20:30:06,990 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T20:30:06,992 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T20:30:06,992 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776206992"}]},"ts":"1733776206992"} 2024-12-09T20:30:06,994 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-09T20:30:06,995 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=6c0df67a61025f1e2ea77a31af80ee4e, ASSIGN}] 2024-12-09T20:30:06,996 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=6c0df67a61025f1e2ea77a31af80ee4e, ASSIGN 2024-12-09T20:30:06,997 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=6c0df67a61025f1e2ea77a31af80ee4e, ASSIGN; state=OFFLINE, location=46f7c943b322,37083,1733776112533; forceNewPlan=false, retain=false 2024-12-09T20:30:07,148 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=6c0df67a61025f1e2ea77a31af80ee4e, regionState=OPENING, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:30:07,151 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=6c0df67a61025f1e2ea77a31af80ee4e, ASSIGN because future has completed 2024-12-09T20:30:07,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c0df67a61025f1e2ea77a31af80ee4e, server=46f7c943b322,37083,1733776112533}] 2024-12-09T20:30:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-09T20:30:07,310 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e. 2024-12-09T20:30:07,310 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c0df67a61025f1e2ea77a31af80ee4e, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e.', STARTKEY => '', ENDKEY => ''} 2024-12-09T20:30:07,311 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 6c0df67a61025f1e2ea77a31af80ee4e 2024-12-09T20:30:07,311 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T20:30:07,311 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 6c0df67a61025f1e2ea77a31af80ee4e 2024-12-09T20:30:07,311 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 6c0df67a61025f1e2ea77a31af80ee4e 2024-12-09T20:30:07,313 INFO [StoreOpener-6c0df67a61025f1e2ea77a31af80ee4e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 6c0df67a61025f1e2ea77a31af80ee4e 2024-12-09T20:30:07,315 INFO [StoreOpener-6c0df67a61025f1e2ea77a31af80ee4e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c0df67a61025f1e2ea77a31af80ee4e columnFamilyName testFamily 2024-12-09T20:30:07,315 DEBUG [StoreOpener-6c0df67a61025f1e2ea77a31af80ee4e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T20:30:07,316 INFO [StoreOpener-6c0df67a61025f1e2ea77a31af80ee4e-1 {}] regionserver.HStore(327): Store=6c0df67a61025f1e2ea77a31af80ee4e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T20:30:07,316 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 6c0df67a61025f1e2ea77a31af80ee4e 2024-12-09T20:30:07,317 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnOpenScannerAsync/6c0df67a61025f1e2ea77a31af80ee4e 2024-12-09T20:30:07,318 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnOpenScannerAsync/6c0df67a61025f1e2ea77a31af80ee4e 2024-12-09T20:30:07,318 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 6c0df67a61025f1e2ea77a31af80ee4e 2024-12-09T20:30:07,318 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 6c0df67a61025f1e2ea77a31af80ee4e 2024-12-09T20:30:07,321 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 6c0df67a61025f1e2ea77a31af80ee4e 2024-12-09T20:30:07,323 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnOpenScannerAsync/6c0df67a61025f1e2ea77a31af80ee4e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T20:30:07,324 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 6c0df67a61025f1e2ea77a31af80ee4e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59349353, jitterRate=-0.11562572419643402}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T20:30:07,324 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6c0df67a61025f1e2ea77a31af80ee4e 2024-12-09T20:30:07,325 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 6c0df67a61025f1e2ea77a31af80ee4e: Running coprocessor pre-open hook at 1733776207311Writing region info on filesystem at 1733776207311Initializing all the Stores at 1733776207312 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733776207313 (+1 ms)Cleaning up temporary data from old regions at 1733776207318 (+5 ms)Running coprocessor post-open hooks at 1733776207324 (+6 ms)Region opened successfully at 1733776207325 (+1 ms) 2024-12-09T20:30:07,326 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e., pid=33, masterSystemTime=1733776207304 2024-12-09T20:30:07,329 DEBUG [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e. 2024-12-09T20:30:07,329 INFO [RS_OPEN_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e. 2024-12-09T20:30:07,330 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=6c0df67a61025f1e2ea77a31af80ee4e, regionState=OPEN, openSeqNum=2, regionLocation=46f7c943b322,37083,1733776112533 2024-12-09T20:30:07,333 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c0df67a61025f1e2ea77a31af80ee4e, server=46f7c943b322,37083,1733776112533 because future has completed 2024-12-09T20:30:07,336 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-09T20:30:07,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 6c0df67a61025f1e2ea77a31af80ee4e, server=46f7c943b322,37083,1733776112533 in 182 msec 2024-12-09T20:30:07,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-09T20:30:07,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=6c0df67a61025f1e2ea77a31af80ee4e, ASSIGN in 342 msec 2024-12-09T20:30:07,340 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T20:30:07,340 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733776207340"}]},"ts":"1733776207340"} 2024-12-09T20:30:07,342 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-09T20:30:07,343 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T20:30:07,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.1850 sec 2024-12-09T20:30:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36137 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-09T20:30:09,182 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-12-09T20:30:09,182 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-09T20:30:09,182 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T20:30:09,187 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-09T20:30:09,187 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T20:30:09,187 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-12-09T20:30:09,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e., hostname=46f7c943b322,37083,1733776112533, seqNum=2] 2024-12-09T20:30:09,199 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-09T20:30:09,199 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-09T20:30:09,201 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-09T20:30:09,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-09T20:30:10,207 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e., hostname=46f7c943b322,37083,1733776112533, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e., hostname=46f7c943b322,37083,1733776112533, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=46f7c943b322:37083 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-12-09T20:30:10,207 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e., hostname=46f7c943b322,37083,1733776112533, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-12-09T20:30:10,207 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e., hostname=46f7c943b322,37083,1733776112533, seqNum=2 from cache 2024-12-09T20:30:10,391 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-12-09T20:30:10,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-12-09T20:30:11,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e., hostname=46f7c943b322,37083,1733776112533, seqNum=2] 2024-12-09T20:30:11,225 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37083 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-09T20:30:12,207 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e., hostname=46f7c943b322,37083,1733776112533, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e., hostname=46f7c943b322,37083,1733776112533, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=46f7c943b322:37083 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=983ms, rpcTimeout=977ms 2024-12-09T20:30:12,207 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e., hostname=46f7c943b322,37083,1733776112533, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=983ms, rpcTimeout=977ms 2024-12-09T20:30:12,207 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e., hostname=46f7c943b322,37083,1733776112533, seqNum=2 from cache 2024-12-09T20:30:12,217 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 223), OpenFileDescriptor=446 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=122 (was 116) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5783 (was 5596) - AvailableMemoryMB LEAK? - 2024-12-09T20:30:12,217 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T20:30:12,217 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T20:30:12,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:30:12,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:30:12,223 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:30:12,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T20:30:12,223 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T20:30:12,223 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T20:30:12,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:30:12,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:30:12,223 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T20:30:12,223 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T20:30:12,224 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1030710501, stopped=false 2024-12-09T20:30:12,224 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=46f7c943b322,36137,1733776111826 2024-12-09T20:30:12,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T20:30:12,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37083-0x10198e9df570001, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T20:30:12,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:30:12,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37083-0x10198e9df570001, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:30:12,226 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T20:30:12,226 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T20:30:12,226 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T20:30:12,226 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:30:12,227 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37083-0x10198e9df570001, quorum=127.0.0.1:61058, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T20:30:12,227 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T20:30:12,227 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '46f7c943b322,37083,1733776112533' ***** 2024-12-09T20:30:12,227 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T20:30:12,227 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T20:30:12,228 INFO [RS:0;46f7c943b322:37083 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T20:30:12,228 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T20:30:12,228 INFO [RS:0;46f7c943b322:37083 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T20:30:12,228 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(3091): Received CLOSE for 977459cd283f0fa2ff2dd4b39ab43f0a 2024-12-09T20:30:12,229 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(3091): Received CLOSE for a81806387c6394fb5229ed8e4f795ee2 2024-12-09T20:30:12,229 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(3091): Received CLOSE for 6c0df67a61025f1e2ea77a31af80ee4e 2024-12-09T20:30:12,229 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(3091): Received CLOSE for d44012135436c54459a77d97b6eda809 2024-12-09T20:30:12,229 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(3091): Received CLOSE for a911b4a9b1ae731dcfe55954949d9033 2024-12-09T20:30:12,229 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(3091): Received CLOSE for d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:30:12,229 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(3091): Received CLOSE for 993e0dbbafe0192b19ca29ebc1a69fd1 2024-12-09T20:30:12,229 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(3091): Received CLOSE for 6e53f96f465f63d860346b347adf8ce9 2024-12-09T20:30:12,229 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 977459cd283f0fa2ff2dd4b39ab43f0a, disabling compactions & flushes 2024-12-09T20:30:12,229 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(3091): Received CLOSE for 04a1637df756cbd7e50b91976c95b02c 2024-12-09T20:30:12,229 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a. 2024-12-09T20:30:12,229 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(3091): Received CLOSE for 8ed7d59ddc232f701aeec84df76a6165 2024-12-09T20:30:12,229 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a. 2024-12-09T20:30:12,229 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a. after waiting 0 ms 2024-12-09T20:30:12,229 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a. 2024-12-09T20:30:12,229 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(959): stopping server 46f7c943b322,37083,1733776112533 2024-12-09T20:30:12,229 INFO [RS:0;46f7c943b322:37083 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T20:30:12,229 INFO [RS:0;46f7c943b322:37083 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;46f7c943b322:37083. 2024-12-09T20:30:12,229 DEBUG [RS:0;46f7c943b322:37083 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T20:30:12,229 DEBUG [RS:0;46f7c943b322:37083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:30:12,230 INFO [RS:0;46f7c943b322:37083 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T20:30:12,230 INFO [RS:0;46f7c943b322:37083 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T20:30:12,230 INFO [RS:0;46f7c943b322:37083 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T20:30:12,230 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T20:30:12,230 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-12-09T20:30:12,230 DEBUG [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T20:30:12,230 INFO [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T20:30:12,230 DEBUG [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T20:30:12,230 DEBUG [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T20:30:12,230 DEBUG [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T20:30:12,230 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 1588230740, 6c0df67a61025f1e2ea77a31af80ee4e, 6e53f96f465f63d860346b347adf8ce9, 8ed7d59ddc232f701aeec84df76a6165, 977459cd283f0fa2ff2dd4b39ab43f0a, 993e0dbbafe0192b19ca29ebc1a69fd1, a81806387c6394fb5229ed8e4f795ee2, a911b4a9b1ae731dcfe55954949d9033, d44012135436c54459a77d97b6eda809, d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:30:12,233 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 977459cd283f0fa2ff2dd4b39ab43f0a 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T20:30:12,233 INFO [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-12-09T20:30:12,243 INFO [regionserver/46f7c943b322:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-09T20:30:12,243 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-12-09T20:30:12,243 INFO [regionserver/46f7c943b322:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-09T20:30:12,243 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-09T20:30:12,290 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnNext/977459cd283f0fa2ff2dd4b39ab43f0a/.tmp/testFamily/fd5f8f2b05454b62a87d19464228ae78 is 61, key is row-0/testFamily:testQualifier/1733776200120/Put/seqid=0 2024-12-09T20:30:12,292 DEBUG [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740/.tmp/info/da748c963ac246c4bed6912dc33bf795 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2./info:regioninfo/1733776146461/Put/seqid=0 2024-12-09T20:30:12,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741855_1031 (size=20564) 2024-12-09T20:30:12,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741856_1032 (size=5049) 2024-12-09T20:30:12,431 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 1588230740, 6c0df67a61025f1e2ea77a31af80ee4e, 6e53f96f465f63d860346b347adf8ce9, 8ed7d59ddc232f701aeec84df76a6165, 977459cd283f0fa2ff2dd4b39ab43f0a, 993e0dbbafe0192b19ca29ebc1a69fd1, a81806387c6394fb5229ed8e4f795ee2, a911b4a9b1ae731dcfe55954949d9033, d44012135436c54459a77d97b6eda809, d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:30:12,631 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 1588230740, 6c0df67a61025f1e2ea77a31af80ee4e, 6e53f96f465f63d860346b347adf8ce9, 8ed7d59ddc232f701aeec84df76a6165, 977459cd283f0fa2ff2dd4b39ab43f0a, 993e0dbbafe0192b19ca29ebc1a69fd1, a81806387c6394fb5229ed8e4f795ee2, a911b4a9b1ae731dcfe55954949d9033, d44012135436c54459a77d97b6eda809, d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:30:12,703 INFO [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740/.tmp/info/da748c963ac246c4bed6912dc33bf795 2024-12-09T20:30:12,703 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnNext/977459cd283f0fa2ff2dd4b39ab43f0a/.tmp/testFamily/fd5f8f2b05454b62a87d19464228ae78 2024-12-09T20:30:12,726 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-09T20:30:12,749 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnNext/977459cd283f0fa2ff2dd4b39ab43f0a/.tmp/testFamily/fd5f8f2b05454b62a87d19464228ae78 as hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnNext/977459cd283f0fa2ff2dd4b39ab43f0a/testFamily/fd5f8f2b05454b62a87d19464228ae78 2024-12-09T20:30:12,759 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnNext/977459cd283f0fa2ff2dd4b39ab43f0a/testFamily/fd5f8f2b05454b62a87d19464228ae78, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T20:30:12,766 DEBUG [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740/.tmp/ns/d89e1c85e2e94d1496ed5de478bde161 is 43, key is default/ns:d/1733776144936/Put/seqid=0 2024-12-09T20:30:12,769 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 977459cd283f0fa2ff2dd4b39ab43f0a in 534ms, sequenceid=8, compaction requested=false 2024-12-09T20:30:12,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741857_1033 (size=5153) 2024-12-09T20:30:12,776 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnNext/977459cd283f0fa2ff2dd4b39ab43f0a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T20:30:12,779 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a. 2024-12-09T20:30:12,779 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 977459cd283f0fa2ff2dd4b39ab43f0a: Waiting for close lock at 1733776212229Running coprocessor pre-close hooks at 1733776212229Disabling compacts and flushes for region at 1733776212229Disabling writes for close at 1733776212229Obtaining lock to block concurrent updates at 1733776212233 (+4 ms)Preparing flush snapshotting stores in 977459cd283f0fa2ff2dd4b39ab43f0a at 1733776212233Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733776212241 (+8 ms)Flushing stores of hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a. at 1733776212242 (+1 ms)Flushing 977459cd283f0fa2ff2dd4b39ab43f0a/testFamily: creating writer at 1733776212243 (+1 ms)Flushing 977459cd283f0fa2ff2dd4b39ab43f0a/testFamily: appending metadata at 1733776212283 (+40 ms)Flushing 977459cd283f0fa2ff2dd4b39ab43f0a/testFamily: closing flushed file at 1733776212287 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6af8d9c6: reopening flushed file at 1733776212747 (+460 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 977459cd283f0fa2ff2dd4b39ab43f0a in 534ms, sequenceid=8, compaction requested=false at 1733776212769 (+22 ms)Writing region close event to WAL at 1733776212771 (+2 ms)Running coprocessor post-close hooks at 1733776212777 (+6 ms)Closed at 1733776212778 (+1 ms) 2024-12-09T20:30:12,779 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1733776197067.977459cd283f0fa2ff2dd4b39ab43f0a. 2024-12-09T20:30:12,779 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a81806387c6394fb5229ed8e4f795ee2, disabling compactions & flushes 2024-12-09T20:30:12,779 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2. 2024-12-09T20:30:12,780 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2. 2024-12-09T20:30:12,780 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2. after waiting 0 ms 2024-12-09T20:30:12,780 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2. 2024-12-09T20:30:12,780 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a81806387c6394fb5229ed8e4f795ee2 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T20:30:12,801 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a81806387c6394fb5229ed8e4f795ee2/.tmp/testFamily/9014c6b02477430a9528afb1f9534435 is 61, key is row-0/testFamily:testQualifier/1733776148330/Put/seqid=0 2024-12-09T20:30:12,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741858_1034 (size=5049) 2024-12-09T20:30:12,812 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T20:30:12,831 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 1588230740, 6c0df67a61025f1e2ea77a31af80ee4e, 6e53f96f465f63d860346b347adf8ce9, 8ed7d59ddc232f701aeec84df76a6165, 993e0dbbafe0192b19ca29ebc1a69fd1, a81806387c6394fb5229ed8e4f795ee2, a911b4a9b1ae731dcfe55954949d9033, d44012135436c54459a77d97b6eda809, d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:30:13,032 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 1588230740, 6c0df67a61025f1e2ea77a31af80ee4e, 6e53f96f465f63d860346b347adf8ce9, 8ed7d59ddc232f701aeec84df76a6165, 993e0dbbafe0192b19ca29ebc1a69fd1, a81806387c6394fb5229ed8e4f795ee2, a911b4a9b1ae731dcfe55954949d9033, d44012135436c54459a77d97b6eda809, d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:30:13,172 INFO [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740/.tmp/ns/d89e1c85e2e94d1496ed5de478bde161 2024-12-09T20:30:13,199 DEBUG [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740/.tmp/table/571a57304fa54358a8dd2d704f17446c is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1733776146475/Put/seqid=0 2024-12-09T20:30:13,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741859_1035 (size=6699) 2024-12-09T20:30:13,205 INFO [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740/.tmp/table/571a57304fa54358a8dd2d704f17446c 2024-12-09T20:30:13,206 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a81806387c6394fb5229ed8e4f795ee2/.tmp/testFamily/9014c6b02477430a9528afb1f9534435 2024-12-09T20:30:13,214 DEBUG [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740/.tmp/info/da748c963ac246c4bed6912dc33bf795 as hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740/info/da748c963ac246c4bed6912dc33bf795 2024-12-09T20:30:13,215 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a81806387c6394fb5229ed8e4f795ee2/.tmp/testFamily/9014c6b02477430a9528afb1f9534435 as hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a81806387c6394fb5229ed8e4f795ee2/testFamily/9014c6b02477430a9528afb1f9534435 2024-12-09T20:30:13,224 INFO [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740/info/da748c963ac246c4bed6912dc33bf795, entries=100, sequenceid=56, filesize=20.1 K 2024-12-09T20:30:13,224 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a81806387c6394fb5229ed8e4f795ee2/testFamily/9014c6b02477430a9528afb1f9534435, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T20:30:13,225 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a81806387c6394fb5229ed8e4f795ee2 in 445ms, sequenceid=8, compaction requested=false 2024-12-09T20:30:13,225 DEBUG [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740/.tmp/ns/d89e1c85e2e94d1496ed5de478bde161 as hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740/ns/d89e1c85e2e94d1496ed5de478bde161 2024-12-09T20:30:13,231 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a81806387c6394fb5229ed8e4f795ee2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T20:30:13,232 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-12-09T20:30:13,232 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 1588230740, 6c0df67a61025f1e2ea77a31af80ee4e, 6e53f96f465f63d860346b347adf8ce9, 8ed7d59ddc232f701aeec84df76a6165, 993e0dbbafe0192b19ca29ebc1a69fd1, a81806387c6394fb5229ed8e4f795ee2, a911b4a9b1ae731dcfe55954949d9033, d44012135436c54459a77d97b6eda809, d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:30:13,232 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2. 2024-12-09T20:30:13,232 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a81806387c6394fb5229ed8e4f795ee2: Waiting for close lock at 1733776212779Running coprocessor pre-close hooks at 1733776212779Disabling compacts and flushes for region at 1733776212779Disabling writes for close at 1733776212780 (+1 ms)Obtaining lock to block concurrent updates at 1733776212780Preparing flush snapshotting stores in a81806387c6394fb5229ed8e4f795ee2 at 1733776212780Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733776212780Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2. at 1733776212781 (+1 ms)Flushing a81806387c6394fb5229ed8e4f795ee2/testFamily: creating writer at 1733776212781Flushing a81806387c6394fb5229ed8e4f795ee2/testFamily: appending metadata at 1733776212801 (+20 ms)Flushing a81806387c6394fb5229ed8e4f795ee2/testFamily: closing flushed file at 1733776212801Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bd8712b: reopening flushed file at 1733776213214 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a81806387c6394fb5229ed8e4f795ee2 in 445ms, sequenceid=8, compaction requested=false at 1733776213225 (+11 ms)Writing region close event to WAL at 1733776213226 (+1 ms)Running coprocessor post-close hooks at 1733776213232 (+6 ms)Closed at 1733776213232 2024-12-09T20:30:13,232 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733776145235.a81806387c6394fb5229ed8e4f795ee2. 2024-12-09T20:30:13,233 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6c0df67a61025f1e2ea77a31af80ee4e, disabling compactions & flushes 2024-12-09T20:30:13,233 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e. 2024-12-09T20:30:13,233 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e. 2024-12-09T20:30:13,233 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e. after waiting 0 ms 2024-12-09T20:30:13,233 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e. 2024-12-09T20:30:13,233 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 6c0df67a61025f1e2ea77a31af80ee4e 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T20:30:13,235 INFO [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740/ns/d89e1c85e2e94d1496ed5de478bde161, entries=2, sequenceid=56, filesize=5.0 K 2024-12-09T20:30:13,237 DEBUG [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740/.tmp/table/571a57304fa54358a8dd2d704f17446c as hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740/table/571a57304fa54358a8dd2d704f17446c 2024-12-09T20:30:13,245 INFO [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740/table/571a57304fa54358a8dd2d704f17446c, entries=20, sequenceid=56, filesize=6.5 K 2024-12-09T20:30:13,246 INFO [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1016ms, sequenceid=56, compaction requested=false 2024-12-09T20:30:13,253 DEBUG [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-12-09T20:30:13,253 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnOpenScannerAsync/6c0df67a61025f1e2ea77a31af80ee4e/.tmp/testFamily/81fe8a09a10c4bf3a02d1b7c7aab2423 is 61, key is row-0/testFamily:testQualifier/1733776209191/Put/seqid=0 2024-12-09T20:30:13,254 DEBUG [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T20:30:13,254 INFO [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T20:30:13,254 DEBUG [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733776212230Running coprocessor pre-close hooks at 1733776212230Disabling compacts and flushes for region at 1733776212230Disabling writes for close at 1733776212230Obtaining lock to block concurrent updates at 1733776212233 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1733776212233Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1733776212241 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733776212242 (+1 ms)Flushing 1588230740/info: creating writer at 1733776212243 (+1 ms)Flushing 1588230740/info: appending metadata at 1733776212289 (+46 ms)Flushing 1588230740/info: closing flushed file at 1733776212289Flushing 1588230740/ns: creating writer at 1733776212747 (+458 ms)Flushing 1588230740/ns: appending metadata at 1733776212765 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1733776212765Flushing 1588230740/table: creating writer at 1733776213181 (+416 ms)Flushing 1588230740/table: appending metadata at 1733776213198 (+17 ms)Flushing 1588230740/table: closing flushed file at 1733776213198Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@769ce69d: reopening flushed file at 1733776213213 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6009dc8c: reopening flushed file at 1733776213224 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28d5889d: reopening flushed file at 1733776213235 (+11 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1016ms, sequenceid=56, compaction requested=false at 1733776213246 (+11 ms)Writing region close event to WAL at 1733776213248 (+2 ms)Running coprocessor post-close hooks at 1733776213254 (+6 ms)Closed at 1733776213254 2024-12-09T20:30:13,255 DEBUG [RS_CLOSE_META-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T20:30:13,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741860_1036 (size=5049) 2024-12-09T20:30:13,432 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 6c0df67a61025f1e2ea77a31af80ee4e, 6e53f96f465f63d860346b347adf8ce9, 8ed7d59ddc232f701aeec84df76a6165, 993e0dbbafe0192b19ca29ebc1a69fd1, a911b4a9b1ae731dcfe55954949d9033, d44012135436c54459a77d97b6eda809, d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:30:13,633 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 6c0df67a61025f1e2ea77a31af80ee4e, 6e53f96f465f63d860346b347adf8ce9, 8ed7d59ddc232f701aeec84df76a6165, 993e0dbbafe0192b19ca29ebc1a69fd1, a911b4a9b1ae731dcfe55954949d9033, d44012135436c54459a77d97b6eda809, d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:30:13,660 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnOpenScannerAsync/6c0df67a61025f1e2ea77a31af80ee4e/.tmp/testFamily/81fe8a09a10c4bf3a02d1b7c7aab2423 2024-12-09T20:30:13,669 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnOpenScannerAsync/6c0df67a61025f1e2ea77a31af80ee4e/.tmp/testFamily/81fe8a09a10c4bf3a02d1b7c7aab2423 as hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnOpenScannerAsync/6c0df67a61025f1e2ea77a31af80ee4e/testFamily/81fe8a09a10c4bf3a02d1b7c7aab2423 2024-12-09T20:30:13,676 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnOpenScannerAsync/6c0df67a61025f1e2ea77a31af80ee4e/testFamily/81fe8a09a10c4bf3a02d1b7c7aab2423, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T20:30:13,677 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6c0df67a61025f1e2ea77a31af80ee4e in 444ms, sequenceid=8, compaction requested=false 2024-12-09T20:30:13,682 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnOpenScannerAsync/6c0df67a61025f1e2ea77a31af80ee4e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T20:30:13,682 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e. 2024-12-09T20:30:13,683 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6c0df67a61025f1e2ea77a31af80ee4e: Waiting for close lock at 1733776213233Running coprocessor pre-close hooks at 1733776213233Disabling compacts and flushes for region at 1733776213233Disabling writes for close at 1733776213233Obtaining lock to block concurrent updates at 1733776213233Preparing flush snapshotting stores in 6c0df67a61025f1e2ea77a31af80ee4e at 1733776213233Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733776213233Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e. at 1733776213234 (+1 ms)Flushing 6c0df67a61025f1e2ea77a31af80ee4e/testFamily: creating writer at 1733776213234Flushing 6c0df67a61025f1e2ea77a31af80ee4e/testFamily: appending metadata at 1733776213253 (+19 ms)Flushing 6c0df67a61025f1e2ea77a31af80ee4e/testFamily: closing flushed file at 1733776213253Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1477fe29: reopening flushed file at 1733776213668 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6c0df67a61025f1e2ea77a31af80ee4e in 444ms, sequenceid=8, compaction requested=false at 1733776213677 (+9 ms)Writing region close event to WAL at 1733776213678 (+1 ms)Running coprocessor post-close hooks at 1733776213682 (+4 ms)Closed at 1733776213682 2024-12-09T20:30:13,683 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733776206158.6c0df67a61025f1e2ea77a31af80ee4e. 2024-12-09T20:30:13,683 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d44012135436c54459a77d97b6eda809, disabling compactions & flushes 2024-12-09T20:30:13,683 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809. 2024-12-09T20:30:13,683 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809. 2024-12-09T20:30:13,683 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809. after waiting 0 ms 2024-12-09T20:30:13,683 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809. 2024-12-09T20:30:13,683 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d44012135436c54459a77d97b6eda809 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T20:30:13,700 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d44012135436c54459a77d97b6eda809/.tmp/testFamily/3649d5195dc945278ee170ddfaf2742c is 61, key is row-0/testFamily:testQualifier/1733776191031/Put/seqid=0 2024-12-09T20:30:13,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741861_1037 (size=5049) 2024-12-09T20:30:13,833 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 6e53f96f465f63d860346b347adf8ce9, 8ed7d59ddc232f701aeec84df76a6165, 993e0dbbafe0192b19ca29ebc1a69fd1, a911b4a9b1ae731dcfe55954949d9033, d44012135436c54459a77d97b6eda809, d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:30:14,033 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 6e53f96f465f63d860346b347adf8ce9, 8ed7d59ddc232f701aeec84df76a6165, 993e0dbbafe0192b19ca29ebc1a69fd1, a911b4a9b1ae731dcfe55954949d9033, d44012135436c54459a77d97b6eda809, d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:30:14,106 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d44012135436c54459a77d97b6eda809/.tmp/testFamily/3649d5195dc945278ee170ddfaf2742c 2024-12-09T20:30:14,115 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d44012135436c54459a77d97b6eda809/.tmp/testFamily/3649d5195dc945278ee170ddfaf2742c as hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d44012135436c54459a77d97b6eda809/testFamily/3649d5195dc945278ee170ddfaf2742c 2024-12-09T20:30:14,121 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d44012135436c54459a77d97b6eda809/testFamily/3649d5195dc945278ee170ddfaf2742c, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T20:30:14,123 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d44012135436c54459a77d97b6eda809 in 440ms, sequenceid=8, compaction requested=false 2024-12-09T20:30:14,127 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/d44012135436c54459a77d97b6eda809/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T20:30:14,128 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809. 2024-12-09T20:30:14,128 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d44012135436c54459a77d97b6eda809: Waiting for close lock at 1733776213683Running coprocessor pre-close hooks at 1733776213683Disabling compacts and flushes for region at 1733776213683Disabling writes for close at 1733776213683Obtaining lock to block concurrent updates at 1733776213683Preparing flush snapshotting stores in d44012135436c54459a77d97b6eda809 at 1733776213683Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733776213683Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809. at 1733776213684 (+1 ms)Flushing d44012135436c54459a77d97b6eda809/testFamily: creating writer at 1733776213684Flushing d44012135436c54459a77d97b6eda809/testFamily: appending metadata at 1733776213700 (+16 ms)Flushing d44012135436c54459a77d97b6eda809/testFamily: closing flushed file at 1733776213700Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b59f0f7: reopening flushed file at 1733776214114 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d44012135436c54459a77d97b6eda809 in 440ms, sequenceid=8, compaction requested=false at 1733776214123 (+9 ms)Writing region close event to WAL at 1733776214123Running coprocessor post-close hooks at 1733776214128 (+5 ms)Closed at 1733776214128 2024-12-09T20:30:14,128 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733776187990.d44012135436c54459a77d97b6eda809. 2024-12-09T20:30:14,129 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a911b4a9b1ae731dcfe55954949d9033, disabling compactions & flushes 2024-12-09T20:30:14,129 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033. 2024-12-09T20:30:14,129 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033. 2024-12-09T20:30:14,129 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033. after waiting 0 ms 2024-12-09T20:30:14,129 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033. 2024-12-09T20:30:14,129 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a911b4a9b1ae731dcfe55954949d9033 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T20:30:14,147 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnOpenScanner/a911b4a9b1ae731dcfe55954949d9033/.tmp/testFamily/c5c2a7f77fb04516ade302e36cb2d5b9 is 61, key is row-0/testFamily:testQualifier/1733776151493/Put/seqid=0 2024-12-09T20:30:14,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741862_1038 (size=5049) 2024-12-09T20:30:14,233 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-12-09T20:30:14,234 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1325): Online Regions={a911b4a9b1ae731dcfe55954949d9033=hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033., d6f662cbb15d92024c09dcbb908e62ba=testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba., 993e0dbbafe0192b19ca29ebc1a69fd1=testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1., 6e53f96f465f63d860346b347adf8ce9=hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9., 04a1637df756cbd7e50b91976c95b02c=testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c., 8ed7d59ddc232f701aeec84df76a6165=testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165.} 2024-12-09T20:30:14,234 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 6e53f96f465f63d860346b347adf8ce9, 8ed7d59ddc232f701aeec84df76a6165, 993e0dbbafe0192b19ca29ebc1a69fd1, a911b4a9b1ae731dcfe55954949d9033, d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:30:14,434 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 6e53f96f465f63d860346b347adf8ce9, 8ed7d59ddc232f701aeec84df76a6165, 993e0dbbafe0192b19ca29ebc1a69fd1, a911b4a9b1ae731dcfe55954949d9033, d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:30:14,552 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnOpenScanner/a911b4a9b1ae731dcfe55954949d9033/.tmp/testFamily/c5c2a7f77fb04516ade302e36cb2d5b9 2024-12-09T20:30:14,561 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnOpenScanner/a911b4a9b1ae731dcfe55954949d9033/.tmp/testFamily/c5c2a7f77fb04516ade302e36cb2d5b9 as hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnOpenScanner/a911b4a9b1ae731dcfe55954949d9033/testFamily/c5c2a7f77fb04516ade302e36cb2d5b9 2024-12-09T20:30:14,568 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnOpenScanner/a911b4a9b1ae731dcfe55954949d9033/testFamily/c5c2a7f77fb04516ade302e36cb2d5b9, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T20:30:14,570 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a911b4a9b1ae731dcfe55954949d9033 in 441ms, sequenceid=8, compaction requested=false 2024-12-09T20:30:14,574 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnOpenScanner/a911b4a9b1ae731dcfe55954949d9033/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T20:30:14,575 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033. 2024-12-09T20:30:14,575 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a911b4a9b1ae731dcfe55954949d9033: Waiting for close lock at 1733776214129Running coprocessor pre-close hooks at 1733776214129Disabling compacts and flushes for region at 1733776214129Disabling writes for close at 1733776214129Obtaining lock to block concurrent updates at 1733776214129Preparing flush snapshotting stores in a911b4a9b1ae731dcfe55954949d9033 at 1733776214129Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733776214129Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033. at 1733776214130 (+1 ms)Flushing a911b4a9b1ae731dcfe55954949d9033/testFamily: creating writer at 1733776214130Flushing a911b4a9b1ae731dcfe55954949d9033/testFamily: appending metadata at 1733776214146 (+16 ms)Flushing a911b4a9b1ae731dcfe55954949d9033/testFamily: closing flushed file at 1733776214146Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3baff44f: reopening flushed file at 1733776214560 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a911b4a9b1ae731dcfe55954949d9033 in 441ms, sequenceid=8, compaction requested=false at 1733776214570 (+10 ms)Writing region close event to WAL at 1733776214570Running coprocessor post-close hooks at 1733776214575 (+5 ms)Closed at 1733776214575 2024-12-09T20:30:14,575 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733776148455.a911b4a9b1ae731dcfe55954949d9033. 2024-12-09T20:30:14,575 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d6f662cbb15d92024c09dcbb908e62ba, disabling compactions & flushes 2024-12-09T20:30:14,575 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba. 2024-12-09T20:30:14,575 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba. 2024-12-09T20:30:14,576 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba. after waiting 1 ms 2024-12-09T20:30:14,576 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba. 2024-12-09T20:30:14,576 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d6f662cbb15d92024c09dcbb908e62ba 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T20:30:14,593 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnNext/d6f662cbb15d92024c09dcbb908e62ba/.tmp/testFamily/e3ea3f9caea5455b8e6900c7bf015696 is 61, key is row-0/testFamily:testQualifier/1733776184952/Put/seqid=0 2024-12-09T20:30:14,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741863_1039 (size=5049) 2024-12-09T20:30:14,634 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 6e53f96f465f63d860346b347adf8ce9, 8ed7d59ddc232f701aeec84df76a6165, 993e0dbbafe0192b19ca29ebc1a69fd1, d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:30:14,834 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 6e53f96f465f63d860346b347adf8ce9, 8ed7d59ddc232f701aeec84df76a6165, 993e0dbbafe0192b19ca29ebc1a69fd1, d6f662cbb15d92024c09dcbb908e62ba 2024-12-09T20:30:14,998 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnNext/d6f662cbb15d92024c09dcbb908e62ba/.tmp/testFamily/e3ea3f9caea5455b8e6900c7bf015696 2024-12-09T20:30:15,008 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnNext/d6f662cbb15d92024c09dcbb908e62ba/.tmp/testFamily/e3ea3f9caea5455b8e6900c7bf015696 as hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnNext/d6f662cbb15d92024c09dcbb908e62ba/testFamily/e3ea3f9caea5455b8e6900c7bf015696 2024-12-09T20:30:15,015 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnNext/d6f662cbb15d92024c09dcbb908e62ba/testFamily/e3ea3f9caea5455b8e6900c7bf015696, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T20:30:15,016 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d6f662cbb15d92024c09dcbb908e62ba in 440ms, sequenceid=8, compaction requested=false 2024-12-09T20:30:15,021 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnNext/d6f662cbb15d92024c09dcbb908e62ba/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T20:30:15,021 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba. 2024-12-09T20:30:15,022 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d6f662cbb15d92024c09dcbb908e62ba: Waiting for close lock at 1733776214575Running coprocessor pre-close hooks at 1733776214575Disabling compacts and flushes for region at 1733776214575Disabling writes for close at 1733776214576 (+1 ms)Obtaining lock to block concurrent updates at 1733776214576Preparing flush snapshotting stores in d6f662cbb15d92024c09dcbb908e62ba at 1733776214576Finished memstore snapshotting testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733776214576Flushing stores of testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba. at 1733776214576Flushing d6f662cbb15d92024c09dcbb908e62ba/testFamily: creating writer at 1733776214577 (+1 ms)Flushing d6f662cbb15d92024c09dcbb908e62ba/testFamily: appending metadata at 1733776214592 (+15 ms)Flushing d6f662cbb15d92024c09dcbb908e62ba/testFamily: closing flushed file at 1733776214592Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43c1b024: reopening flushed file at 1733776215007 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d6f662cbb15d92024c09dcbb908e62ba in 440ms, sequenceid=8, compaction requested=false at 1733776215016 (+9 ms)Writing region close event to WAL at 1733776215017 (+1 ms)Running coprocessor post-close hooks at 1733776215021 (+4 ms)Closed at 1733776215021 2024-12-09T20:30:15,022 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1733776181906.d6f662cbb15d92024c09dcbb908e62ba. 2024-12-09T20:30:15,022 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 993e0dbbafe0192b19ca29ebc1a69fd1, disabling compactions & flushes 2024-12-09T20:30:15,022 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1. 2024-12-09T20:30:15,022 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1. 2024-12-09T20:30:15,022 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1. after waiting 0 ms 2024-12-09T20:30:15,022 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1. 2024-12-09T20:30:15,022 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 993e0dbbafe0192b19ca29ebc1a69fd1 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T20:30:15,035 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 6e53f96f465f63d860346b347adf8ce9, 8ed7d59ddc232f701aeec84df76a6165, 993e0dbbafe0192b19ca29ebc1a69fd1 2024-12-09T20:30:15,040 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnOpenScanner/993e0dbbafe0192b19ca29ebc1a69fd1/.tmp/testFamily/48d0bb58342242478e4d315cdd2bc17b is 61, key is row-0/testFamily:testQualifier/1733776172782/Put/seqid=0 2024-12-09T20:30:15,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741864_1040 (size=5049) 2024-12-09T20:30:15,235 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-12-09T20:30:15,235 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1325): Online Regions={993e0dbbafe0192b19ca29ebc1a69fd1=testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1., 6e53f96f465f63d860346b347adf8ce9=hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9., 04a1637df756cbd7e50b91976c95b02c=testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c., 8ed7d59ddc232f701aeec84df76a6165=testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165.} 2024-12-09T20:30:15,235 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 6e53f96f465f63d860346b347adf8ce9, 8ed7d59ddc232f701aeec84df76a6165, 993e0dbbafe0192b19ca29ebc1a69fd1 2024-12-09T20:30:15,435 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 6e53f96f465f63d860346b347adf8ce9, 8ed7d59ddc232f701aeec84df76a6165, 993e0dbbafe0192b19ca29ebc1a69fd1 2024-12-09T20:30:15,446 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnOpenScanner/993e0dbbafe0192b19ca29ebc1a69fd1/.tmp/testFamily/48d0bb58342242478e4d315cdd2bc17b 2024-12-09T20:30:15,454 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnOpenScanner/993e0dbbafe0192b19ca29ebc1a69fd1/.tmp/testFamily/48d0bb58342242478e4d315cdd2bc17b as hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnOpenScanner/993e0dbbafe0192b19ca29ebc1a69fd1/testFamily/48d0bb58342242478e4d315cdd2bc17b 2024-12-09T20:30:15,461 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnOpenScanner/993e0dbbafe0192b19ca29ebc1a69fd1/testFamily/48d0bb58342242478e4d315cdd2bc17b, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T20:30:15,462 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 993e0dbbafe0192b19ca29ebc1a69fd1 in 440ms, sequenceid=8, compaction requested=false 2024-12-09T20:30:15,467 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnOpenScanner/993e0dbbafe0192b19ca29ebc1a69fd1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T20:30:15,467 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1. 2024-12-09T20:30:15,467 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 993e0dbbafe0192b19ca29ebc1a69fd1: Waiting for close lock at 1733776215022Running coprocessor pre-close hooks at 1733776215022Disabling compacts and flushes for region at 1733776215022Disabling writes for close at 1733776215022Obtaining lock to block concurrent updates at 1733776215022Preparing flush snapshotting stores in 993e0dbbafe0192b19ca29ebc1a69fd1 at 1733776215022Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733776215023 (+1 ms)Flushing stores of testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1. at 1733776215023Flushing 993e0dbbafe0192b19ca29ebc1a69fd1/testFamily: creating writer at 1733776215023Flushing 993e0dbbafe0192b19ca29ebc1a69fd1/testFamily: appending metadata at 1733776215040 (+17 ms)Flushing 993e0dbbafe0192b19ca29ebc1a69fd1/testFamily: closing flushed file at 1733776215040Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29e9abf7: reopening flushed file at 1733776215453 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 993e0dbbafe0192b19ca29ebc1a69fd1 in 440ms, sequenceid=8, compaction requested=false at 1733776215462 (+9 ms)Writing region close event to WAL at 1733776215463 (+1 ms)Running coprocessor post-close hooks at 1733776215467 (+4 ms)Closed at 1733776215467 2024-12-09T20:30:15,467 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1733776169740.993e0dbbafe0192b19ca29ebc1a69fd1. 2024-12-09T20:30:15,468 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6e53f96f465f63d860346b347adf8ce9, disabling compactions & flushes 2024-12-09T20:30:15,468 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9. 2024-12-09T20:30:15,468 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9. 2024-12-09T20:30:15,468 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9. after waiting 0 ms 2024-12-09T20:30:15,468 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9. 2024-12-09T20:30:15,468 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 6e53f96f465f63d860346b347adf8ce9 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T20:30:15,485 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnNextAsync/6e53f96f465f63d860346b347adf8ce9/.tmp/testFamily/cbf917e0a1a6482ba172f338411a7a01 is 61, key is row-0/testFamily:testQualifier/1733776160583/Put/seqid=0 2024-12-09T20:30:15,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741865_1041 (size=5049) 2024-12-09T20:30:15,636 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 6e53f96f465f63d860346b347adf8ce9, 8ed7d59ddc232f701aeec84df76a6165 2024-12-09T20:30:15,836 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 6e53f96f465f63d860346b347adf8ce9, 8ed7d59ddc232f701aeec84df76a6165 2024-12-09T20:30:15,890 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnNextAsync/6e53f96f465f63d860346b347adf8ce9/.tmp/testFamily/cbf917e0a1a6482ba172f338411a7a01 2024-12-09T20:30:15,898 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnNextAsync/6e53f96f465f63d860346b347adf8ce9/.tmp/testFamily/cbf917e0a1a6482ba172f338411a7a01 as hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnNextAsync/6e53f96f465f63d860346b347adf8ce9/testFamily/cbf917e0a1a6482ba172f338411a7a01 2024-12-09T20:30:15,905 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnNextAsync/6e53f96f465f63d860346b347adf8ce9/testFamily/cbf917e0a1a6482ba172f338411a7a01, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T20:30:15,906 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6e53f96f465f63d860346b347adf8ce9 in 438ms, sequenceid=8, compaction requested=false 2024-12-09T20:30:15,911 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/hbase/testMetaScanTimeoutOnNextAsync/6e53f96f465f63d860346b347adf8ce9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T20:30:15,911 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9. 2024-12-09T20:30:15,911 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6e53f96f465f63d860346b347adf8ce9: Waiting for close lock at 1733776215468Running coprocessor pre-close hooks at 1733776215468Disabling compacts and flushes for region at 1733776215468Disabling writes for close at 1733776215468Obtaining lock to block concurrent updates at 1733776215468Preparing flush snapshotting stores in 6e53f96f465f63d860346b347adf8ce9 at 1733776215468Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733776215468Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9. at 1733776215469 (+1 ms)Flushing 6e53f96f465f63d860346b347adf8ce9/testFamily: creating writer at 1733776215469Flushing 6e53f96f465f63d860346b347adf8ce9/testFamily: appending metadata at 1733776215485 (+16 ms)Flushing 6e53f96f465f63d860346b347adf8ce9/testFamily: closing flushed file at 1733776215485Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55f48ef7: reopening flushed file at 1733776215897 (+412 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6e53f96f465f63d860346b347adf8ce9 in 438ms, sequenceid=8, compaction requested=false at 1733776215906 (+9 ms)Writing region close event to WAL at 1733776215907 (+1 ms)Running coprocessor post-close hooks at 1733776215911 (+4 ms)Closed at 1733776215911 2024-12-09T20:30:15,912 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733776157548.6e53f96f465f63d860346b347adf8ce9. 2024-12-09T20:30:15,912 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 04a1637df756cbd7e50b91976c95b02c, disabling compactions & flushes 2024-12-09T20:30:15,912 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c. 2024-12-09T20:30:15,912 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c. 2024-12-09T20:30:15,912 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c. after waiting 0 ms 2024-12-09T20:30:15,912 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c. 2024-12-09T20:30:15,912 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 04a1637df756cbd7e50b91976c95b02c 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T20:30:15,929 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testRetryOutOfOrderScannerNextException/04a1637df756cbd7e50b91976c95b02c/.tmp/testFamily/63d15075a90741558c257b8974eda9f1 is 61, key is row-0/testFamily:testQualifier/1733776169675/Put/seqid=0 2024-12-09T20:30:15,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741866_1042 (size=5049) 2024-12-09T20:30:16,036 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 8ed7d59ddc232f701aeec84df76a6165 2024-12-09T20:30:16,236 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-09T20:30:16,236 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1325): Online Regions={04a1637df756cbd7e50b91976c95b02c=testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c., 8ed7d59ddc232f701aeec84df76a6165=testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165.} 2024-12-09T20:30:16,237 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 04a1637df756cbd7e50b91976c95b02c, 8ed7d59ddc232f701aeec84df76a6165 2024-12-09T20:30:16,335 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testRetryOutOfOrderScannerNextException/04a1637df756cbd7e50b91976c95b02c/.tmp/testFamily/63d15075a90741558c257b8974eda9f1 2024-12-09T20:30:16,343 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testRetryOutOfOrderScannerNextException/04a1637df756cbd7e50b91976c95b02c/.tmp/testFamily/63d15075a90741558c257b8974eda9f1 as hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testRetryOutOfOrderScannerNextException/04a1637df756cbd7e50b91976c95b02c/testFamily/63d15075a90741558c257b8974eda9f1 2024-12-09T20:30:16,350 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testRetryOutOfOrderScannerNextException/04a1637df756cbd7e50b91976c95b02c/testFamily/63d15075a90741558c257b8974eda9f1, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T20:30:16,351 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 04a1637df756cbd7e50b91976c95b02c in 439ms, sequenceid=8, compaction requested=false 2024-12-09T20:30:16,355 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testRetryOutOfOrderScannerNextException/04a1637df756cbd7e50b91976c95b02c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T20:30:16,356 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c. 2024-12-09T20:30:16,356 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 04a1637df756cbd7e50b91976c95b02c: Waiting for close lock at 1733776215912Running coprocessor pre-close hooks at 1733776215912Disabling compacts and flushes for region at 1733776215912Disabling writes for close at 1733776215912Obtaining lock to block concurrent updates at 1733776215912Preparing flush snapshotting stores in 04a1637df756cbd7e50b91976c95b02c at 1733776215912Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733776215912Flushing stores of testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c. at 1733776215913 (+1 ms)Flushing 04a1637df756cbd7e50b91976c95b02c/testFamily: creating writer at 1733776215913Flushing 04a1637df756cbd7e50b91976c95b02c/testFamily: appending metadata at 1733776215929 (+16 ms)Flushing 04a1637df756cbd7e50b91976c95b02c/testFamily: closing flushed file at 1733776215929Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a48a8d5: reopening flushed file at 1733776216342 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 04a1637df756cbd7e50b91976c95b02c in 439ms, sequenceid=8, compaction requested=false at 1733776216351 (+9 ms)Writing region close event to WAL at 1733776216352 (+1 ms)Running coprocessor post-close hooks at 1733776216356 (+4 ms)Closed at 1733776216356 2024-12-09T20:30:16,356 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1733776166634.04a1637df756cbd7e50b91976c95b02c. 2024-12-09T20:30:16,357 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8ed7d59ddc232f701aeec84df76a6165, disabling compactions & flushes 2024-12-09T20:30:16,357 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165. 2024-12-09T20:30:16,357 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165. 2024-12-09T20:30:16,357 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165. after waiting 0 ms 2024-12-09T20:30:16,357 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165. 2024-12-09T20:30:16,357 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 8ed7d59ddc232f701aeec84df76a6165 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-09T20:30:16,373 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnNextAsync/8ed7d59ddc232f701aeec84df76a6165/.tmp/testFamily/25975da605674a7d9f0031a98051b3fb is 61, key is row-0/testFamily:testQualifier/1733776178863/Put/seqid=0 2024-12-09T20:30:16,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741867_1043 (size=5049) 2024-12-09T20:30:16,437 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 8ed7d59ddc232f701aeec84df76a6165 2024-12-09T20:30:16,637 DEBUG [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1351): Waiting on 8ed7d59ddc232f701aeec84df76a6165 2024-12-09T20:30:16,778 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnNextAsync/8ed7d59ddc232f701aeec84df76a6165/.tmp/testFamily/25975da605674a7d9f0031a98051b3fb 2024-12-09T20:30:16,787 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnNextAsync/8ed7d59ddc232f701aeec84df76a6165/.tmp/testFamily/25975da605674a7d9f0031a98051b3fb as hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnNextAsync/8ed7d59ddc232f701aeec84df76a6165/testFamily/25975da605674a7d9f0031a98051b3fb 2024-12-09T20:30:16,793 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnNextAsync/8ed7d59ddc232f701aeec84df76a6165/testFamily/25975da605674a7d9f0031a98051b3fb, entries=4, sequenceid=8, filesize=4.9 K 2024-12-09T20:30:16,794 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8ed7d59ddc232f701aeec84df76a6165 in 437ms, sequenceid=8, compaction requested=false 2024-12-09T20:30:16,799 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/data/default/testNormalScanTimeoutOnNextAsync/8ed7d59ddc232f701aeec84df76a6165/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-09T20:30:16,799 INFO [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165. 2024-12-09T20:30:16,799 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8ed7d59ddc232f701aeec84df76a6165: Waiting for close lock at 1733776216356Running coprocessor pre-close hooks at 1733776216356Disabling compacts and flushes for region at 1733776216356Disabling writes for close at 1733776216357 (+1 ms)Obtaining lock to block concurrent updates at 1733776216357Preparing flush snapshotting stores in 8ed7d59ddc232f701aeec84df76a6165 at 1733776216357Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733776216357Flushing stores of testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165. at 1733776216357Flushing 8ed7d59ddc232f701aeec84df76a6165/testFamily: creating writer at 1733776216358 (+1 ms)Flushing 8ed7d59ddc232f701aeec84df76a6165/testFamily: appending metadata at 1733776216373 (+15 ms)Flushing 8ed7d59ddc232f701aeec84df76a6165/testFamily: closing flushed file at 1733776216373Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cf8515: reopening flushed file at 1733776216786 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8ed7d59ddc232f701aeec84df76a6165 in 437ms, sequenceid=8, compaction requested=false at 1733776216794 (+8 ms)Writing region close event to WAL at 1733776216795 (+1 ms)Running coprocessor post-close hooks at 1733776216799 (+4 ms)Closed at 1733776216799 2024-12-09T20:30:16,800 DEBUG [RS_CLOSE_REGION-regionserver/46f7c943b322:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1733776175817.8ed7d59ddc232f701aeec84df76a6165. 2024-12-09T20:30:16,837 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(976): stopping server 46f7c943b322,37083,1733776112533; all regions closed. 2024-12-09T20:30:16,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741834_1010 (size=19479) 2024-12-09T20:30:16,848 DEBUG [RS:0;46f7c943b322:37083 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/oldWALs 2024-12-09T20:30:16,848 INFO [RS:0;46f7c943b322:37083 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 46f7c943b322%2C37083%2C1733776112533.meta:.meta(num 1733776144778) 2024-12-09T20:30:16,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741833_1009 (size=20031) 2024-12-09T20:30:16,853 DEBUG [RS:0;46f7c943b322:37083 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/oldWALs 2024-12-09T20:30:16,853 INFO [RS:0;46f7c943b322:37083 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 46f7c943b322%2C37083%2C1733776112533:(num 1733776144317) 2024-12-09T20:30:16,853 DEBUG [RS:0;46f7c943b322:37083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T20:30:16,853 INFO [RS:0;46f7c943b322:37083 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T20:30:16,853 INFO [RS:0;46f7c943b322:37083 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T20:30:16,853 INFO [RS:0;46f7c943b322:37083 {}] hbase.ChoreService(370): Chore service for: regionserver/46f7c943b322:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T20:30:16,853 INFO [RS:0;46f7c943b322:37083 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T20:30:16,854 INFO [regionserver/46f7c943b322:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T20:30:16,855 INFO [RS:0;46f7c943b322:37083 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37083 2024-12-09T20:30:16,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37083-0x10198e9df570001, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/46f7c943b322,37083,1733776112533 2024-12-09T20:30:16,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T20:30:16,858 INFO [RS:0;46f7c943b322:37083 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T20:30:16,863 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [46f7c943b322,37083,1733776112533] 2024-12-09T20:30:16,864 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/46f7c943b322,37083,1733776112533 already deleted, retry=false 2024-12-09T20:30:16,864 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 46f7c943b322,37083,1733776112533 expired; onlineServers=0 2024-12-09T20:30:16,865 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '46f7c943b322,36137,1733776111826' ***** 2024-12-09T20:30:16,865 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T20:30:16,865 INFO [M:0;46f7c943b322:36137 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T20:30:16,865 INFO [M:0;46f7c943b322:36137 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T20:30:16,865 DEBUG [M:0;46f7c943b322:36137 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T20:30:16,865 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T20:30:16,865 DEBUG [M:0;46f7c943b322:36137 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T20:30:16,865 DEBUG [master/46f7c943b322:0:becomeActiveMaster-HFileCleaner.small.0-1733776114117 {}] cleaner.HFileCleaner(306): Exit Thread[master/46f7c943b322:0:becomeActiveMaster-HFileCleaner.small.0-1733776114117,5,FailOnTimeoutGroup] 2024-12-09T20:30:16,865 DEBUG [master/46f7c943b322:0:becomeActiveMaster-HFileCleaner.large.0-1733776114116 {}] cleaner.HFileCleaner(306): Exit Thread[master/46f7c943b322:0:becomeActiveMaster-HFileCleaner.large.0-1733776114116,5,FailOnTimeoutGroup] 2024-12-09T20:30:16,865 INFO [M:0;46f7c943b322:36137 {}] hbase.ChoreService(370): Chore service for: master/46f7c943b322:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T20:30:16,866 INFO [M:0;46f7c943b322:36137 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T20:30:16,866 DEBUG [M:0;46f7c943b322:36137 {}] master.HMaster(1795): Stopping service threads 2024-12-09T20:30:16,866 INFO [M:0;46f7c943b322:36137 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T20:30:16,866 INFO [M:0;46f7c943b322:36137 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T20:30:16,867 INFO [M:0;46f7c943b322:36137 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T20:30:16,867 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T20:30:16,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T20:30:16,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T20:30:16,867 DEBUG [M:0;46f7c943b322:36137 {}] zookeeper.ZKUtil(347): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T20:30:16,867 WARN [M:0;46f7c943b322:36137 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T20:30:16,868 INFO [M:0;46f7c943b322:36137 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/.lastflushedseqids 2024-12-09T20:30:16,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741868_1044 (size=649) 2024-12-09T20:30:16,923 INFO [regionserver/46f7c943b322:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T20:30:16,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37083-0x10198e9df570001, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T20:30:16,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37083-0x10198e9df570001, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T20:30:16,964 INFO [RS:0;46f7c943b322:37083 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T20:30:16,964 INFO [RS:0;46f7c943b322:37083 {}] regionserver.HRegionServer(1031): Exiting; stopping=46f7c943b322,37083,1733776112533; zookeeper connection closed. 2024-12-09T20:30:16,964 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4cc3e69 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4cc3e69 2024-12-09T20:30:16,965 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-09T20:30:17,281 INFO [M:0;46f7c943b322:36137 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T20:30:17,282 INFO [M:0;46f7c943b322:36137 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T20:30:17,282 DEBUG [M:0;46f7c943b322:36137 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T20:30:17,282 INFO [M:0;46f7c943b322:36137 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T20:30:17,282 DEBUG [M:0;46f7c943b322:36137 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T20:30:17,282 DEBUG [M:0;46f7c943b322:36137 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T20:30:17,282 DEBUG [M:0;46f7c943b322:36137 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T20:30:17,282 INFO [M:0;46f7c943b322:36137 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=162.97 KB heapSize=190.86 KB 2024-12-09T20:30:17,300 DEBUG [M:0;46f7c943b322:36137 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9964ae7b361e4854a9c909c629e6f0ca is 82, key is hbase:meta,,1/info:regioninfo/1733776144864/Put/seqid=0 2024-12-09T20:30:17,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741869_1045 (size=5672) 2024-12-09T20:30:17,706 INFO [M:0;46f7c943b322:36137 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9964ae7b361e4854a9c909c629e6f0ca 2024-12-09T20:30:17,731 DEBUG [M:0;46f7c943b322:36137 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2564f66e71743bca14f6b0930224e92 is 811, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733776146482/Put/seqid=0 2024-12-09T20:30:17,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741870_1046 (size=14068) 2024-12-09T20:30:18,137 INFO [M:0;46f7c943b322:36137 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.41 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2564f66e71743bca14f6b0930224e92 2024-12-09T20:30:18,144 INFO [M:0;46f7c943b322:36137 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c2564f66e71743bca14f6b0930224e92 2024-12-09T20:30:18,161 DEBUG [M:0;46f7c943b322:36137 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7ed16dda719947a1bed8ef4d1b93db7c is 69, key is 46f7c943b322,37083,1733776112533/rs:state/1733776114162/Put/seqid=0 2024-12-09T20:30:18,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741871_1047 (size=5156) 2024-12-09T20:30:18,280 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T20:30:18,567 INFO [M:0;46f7c943b322:36137 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7ed16dda719947a1bed8ef4d1b93db7c 2024-12-09T20:30:18,574 DEBUG [M:0;46f7c943b322:36137 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9964ae7b361e4854a9c909c629e6f0ca as hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9964ae7b361e4854a9c909c629e6f0ca 2024-12-09T20:30:18,580 INFO [M:0;46f7c943b322:36137 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9964ae7b361e4854a9c909c629e6f0ca, entries=8, sequenceid=317, filesize=5.5 K 2024-12-09T20:30:18,582 DEBUG [M:0;46f7c943b322:36137 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c2564f66e71743bca14f6b0930224e92 as hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c2564f66e71743bca14f6b0930224e92 2024-12-09T20:30:18,587 INFO [M:0;46f7c943b322:36137 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c2564f66e71743bca14f6b0930224e92 2024-12-09T20:30:18,587 INFO [M:0;46f7c943b322:36137 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c2564f66e71743bca14f6b0930224e92, entries=33, sequenceid=317, filesize=13.7 K 2024-12-09T20:30:18,589 DEBUG [M:0;46f7c943b322:36137 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7ed16dda719947a1bed8ef4d1b93db7c as hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7ed16dda719947a1bed8ef4d1b93db7c 2024-12-09T20:30:18,594 INFO [M:0;46f7c943b322:36137 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40965/user/jenkins/test-data/b33e5289-abed-c07f-e87e-7d429b13cf05/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7ed16dda719947a1bed8ef4d1b93db7c, entries=1, sequenceid=317, filesize=5.0 K 2024-12-09T20:30:18,596 INFO [M:0;46f7c943b322:36137 {}] regionserver.HRegion(3140): Finished flush of dataSize ~162.97 KB/166879, heapSize ~190.56 KB/195136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1313ms, sequenceid=317, compaction requested=false 2024-12-09T20:30:18,597 INFO [M:0;46f7c943b322:36137 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T20:30:18,597 DEBUG [M:0;46f7c943b322:36137 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733776217282Disabling compacts and flushes for region at 1733776217282Disabling writes for close at 1733776217282Obtaining lock to block concurrent updates at 1733776217282Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733776217282Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166879, getHeapSize=195376, getOffHeapSize=0, getCellsCount=364 at 1733776217283 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733776217283Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733776217283Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733776217300 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733776217300Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733776217713 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733776217731 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733776217731Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733776218144 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733776218160 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733776218161 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ef61254: reopening flushed file at 1733776218573 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14a8171d: reopening flushed file at 1733776218581 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33b799cd: reopening flushed file at 1733776218588 (+7 ms)Finished flush of dataSize ~162.97 KB/166879, heapSize ~190.56 KB/195136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1313ms, sequenceid=317, compaction requested=false at 1733776218596 (+8 ms)Writing region close event to WAL at 1733776218597 (+1 ms)Closed at 1733776218597 2024-12-09T20:30:18,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41477 is added to blk_1073741830_1006 (size=188091) 2024-12-09T20:30:18,601 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T20:30:18,601 INFO [M:0;46f7c943b322:36137 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T20:30:18,601 INFO [M:0;46f7c943b322:36137 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36137 2024-12-09T20:30:18,601 INFO [M:0;46f7c943b322:36137 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T20:30:18,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T20:30:18,704 INFO [M:0;46f7c943b322:36137 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T20:30:18,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36137-0x10198e9df570000, quorum=127.0.0.1:61058, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T20:30:18,709 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5976fb41{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T20:30:18,711 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T20:30:18,711 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T20:30:18,711 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T20:30:18,711 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/hadoop.log.dir/,STOPPED} 2024-12-09T20:30:18,714 WARN [BP-2093874798-172.17.0.2-1733776108440 heartbeating to localhost/127.0.0.1:40965 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T20:30:18,714 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T20:30:18,714 WARN [BP-2093874798-172.17.0.2-1733776108440 heartbeating to localhost/127.0.0.1:40965 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2093874798-172.17.0.2-1733776108440 (Datanode Uuid b5d3d332-450a-40fa-aaa9-0c2f15522f69) service to localhost/127.0.0.1:40965 2024-12-09T20:30:18,714 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T20:30:18,716 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/cluster_0bfa8ede-61a6-2250-c0ec-ce485b9e5181/data/data1/current/BP-2093874798-172.17.0.2-1733776108440 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T20:30:18,716 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/cluster_0bfa8ede-61a6-2250-c0ec-ce485b9e5181/data/data2/current/BP-2093874798-172.17.0.2-1733776108440 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T20:30:18,716 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T20:30:18,725 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@eb30bc1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T20:30:18,726 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T20:30:18,726 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T20:30:18,726 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T20:30:18,726 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dba0461d-377e-2d1d-7e59-3e43e6b67e60/hadoop.log.dir/,STOPPED} 2024-12-09T20:30:18,734 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T20:30:18,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down