2024-11-13 17:58:02,877 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-13 17:58:02,936 main DEBUG Took 0.044220 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 17:58:02,937 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 17:58:02,937 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 17:58:02,938 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 17:58:02,940 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 17:58:02,967 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 17:58:03,008 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 17:58:03,010 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 17:58:03,011 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 17:58:03,012 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 17:58:03,012 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 17:58:03,013 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 17:58:03,014 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 17:58:03,015 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 17:58:03,015 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 17:58:03,016 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 17:58:03,017 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 17:58:03,017 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 17:58:03,018 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 17:58:03,018 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 17:58:03,019 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 17:58:03,019 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 17:58:03,021 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 17:58:03,021 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 17:58:03,022 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 17:58:03,022 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 17:58:03,023 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 17:58:03,023 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 17:58:03,024 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 17:58:03,024 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 17:58:03,025 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 17:58:03,025 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 17:58:03,028 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 17:58:03,037 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 17:58:03,047 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 17:58:03,062 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 17:58:03,065 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 17:58:03,065 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 17:58:03,079 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 17:58:03,091 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 17:58:03,094 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 17:58:03,094 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 17:58:03,095 main DEBUG createAppenders(={Console}) 2024-11-13 17:58:03,096 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b initialized 2024-11-13 17:58:03,097 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-13 17:58:03,097 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b OK. 2024-11-13 17:58:03,098 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 17:58:03,098 main DEBUG OutputStream closed 2024-11-13 17:58:03,099 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 17:58:03,099 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 17:58:03,099 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@37d4349f OK 2024-11-13 17:58:03,268 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 17:58:03,271 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 17:58:03,273 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 17:58:03,274 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 17:58:03,274 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 17:58:03,275 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 17:58:03,275 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 17:58:03,276 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 17:58:03,276 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 17:58:03,277 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 17:58:03,277 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 17:58:03,278 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 17:58:03,278 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 17:58:03,279 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 17:58:03,279 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 17:58:03,279 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 17:58:03,280 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 17:58:03,281 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 17:58:03,284 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 17:58:03,284 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@29ca3d04) with optional ClassLoader: null 2024-11-13 17:58:03,285 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 17:58:03,289 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@29ca3d04] started OK. 2024-11-13T17:58:03,312 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-13 17:58:03,318 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 17:58:03,318 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T17:58:03,726 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e 2024-11-13T17:58:03,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T17:58:03,843 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/cluster_97dbc57c-85a5-456c-2cc5-cce186832a72, deleteOnExit=true 2024-11-13T17:58:03,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T17:58:03,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/test.cache.data in system properties and HBase conf 2024-11-13T17:58:03,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T17:58:03,848 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/hadoop.log.dir in system properties and HBase conf 2024-11-13T17:58:03,849 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T17:58:03,850 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T17:58:03,850 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T17:58:04,052 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T17:58:04,263 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T17:58:04,269 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T17:58:04,273 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T17:58:04,274 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T17:58:04,275 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T17:58:04,275 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T17:58:04,276 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T17:58:04,276 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T17:58:04,277 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T17:58:04,278 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T17:58:04,278 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/nfs.dump.dir in system properties and HBase conf 2024-11-13T17:58:04,279 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/java.io.tmpdir in system properties and HBase conf 2024-11-13T17:58:04,279 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T17:58:04,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T17:58:04,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T17:58:05,403 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T17:58:05,494 INFO [Time-limited test {}] log.Log(170): Logging initialized @3727ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T17:58:05,613 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T17:58:05,693 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T17:58:05,735 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T17:58:05,736 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T17:58:05,738 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T17:58:05,755 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T17:58:05,759 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3653a146{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/hadoop.log.dir/,AVAILABLE} 2024-11-13T17:58:05,760 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5c1e5f71{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T17:58:05,990 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4df87c21{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/java.io.tmpdir/jetty-localhost-35915-hadoop-hdfs-3_4_1-tests_jar-_-any-8671029352690614570/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T17:58:06,002 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@29047fa7{HTTP/1.1, (http/1.1)}{localhost:35915} 2024-11-13T17:58:06,002 INFO [Time-limited test {}] server.Server(415): Started @4237ms 2024-11-13T17:58:06,505 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T17:58:06,516 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T17:58:06,518 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T17:58:06,518 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T17:58:06,518 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T17:58:06,520 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@23c598c8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/hadoop.log.dir/,AVAILABLE} 2024-11-13T17:58:06,521 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a781d84{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T17:58:06,656 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@16d0bc3d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/java.io.tmpdir/jetty-localhost-39401-hadoop-hdfs-3_4_1-tests_jar-_-any-1548241528340199828/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T17:58:06,657 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2e1cc352{HTTP/1.1, (http/1.1)}{localhost:39401} 2024-11-13T17:58:06,658 INFO [Time-limited test {}] server.Server(415): Started @4892ms 2024-11-13T17:58:06,733 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T17:58:07,382 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/cluster_97dbc57c-85a5-456c-2cc5-cce186832a72/data/data1/current/BP-2066668531-172.17.0.3-1731520685052/current, will proceed with Du for space computation calculation, 2024-11-13T17:58:07,383 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/cluster_97dbc57c-85a5-456c-2cc5-cce186832a72/data/data2/current/BP-2066668531-172.17.0.3-1731520685052/current, will proceed with Du for space computation calculation, 2024-11-13T17:58:07,512 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T17:58:07,607 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4bc6046b70fb41f5 with lease ID 0x82c8fb52fd7c2835: Processing first storage report for DS-5f126945-7071-47ac-9261-62e668c3d3ed from datanode DatanodeRegistration(127.0.0.1:39637, datanodeUuid=a1c690e6-c995-4d71-94ea-97556a439951, infoPort=33155, infoSecurePort=0, ipcPort=43583, storageInfo=lv=-57;cid=testClusterID;nsid=476081213;c=1731520685052) 2024-11-13T17:58:07,608 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4bc6046b70fb41f5 with lease ID 0x82c8fb52fd7c2835: from storage DS-5f126945-7071-47ac-9261-62e668c3d3ed node DatanodeRegistration(127.0.0.1:39637, datanodeUuid=a1c690e6-c995-4d71-94ea-97556a439951, infoPort=33155, infoSecurePort=0, ipcPort=43583, storageInfo=lv=-57;cid=testClusterID;nsid=476081213;c=1731520685052), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-13T17:58:07,610 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4bc6046b70fb41f5 with lease ID 0x82c8fb52fd7c2835: Processing first storage report for DS-c0aa2c14-2d42-432b-871f-2a4349ef0cc5 from datanode DatanodeRegistration(127.0.0.1:39637, datanodeUuid=a1c690e6-c995-4d71-94ea-97556a439951, infoPort=33155, infoSecurePort=0, ipcPort=43583, storageInfo=lv=-57;cid=testClusterID;nsid=476081213;c=1731520685052) 2024-11-13T17:58:07,610 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4bc6046b70fb41f5 with lease ID 0x82c8fb52fd7c2835: from storage DS-c0aa2c14-2d42-432b-871f-2a4349ef0cc5 node DatanodeRegistration(127.0.0.1:39637, datanodeUuid=a1c690e6-c995-4d71-94ea-97556a439951, infoPort=33155, infoSecurePort=0, ipcPort=43583, storageInfo=lv=-57;cid=testClusterID;nsid=476081213;c=1731520685052), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T17:58:07,678 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e 2024-11-13T17:58:07,816 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/cluster_97dbc57c-85a5-456c-2cc5-cce186832a72/zookeeper_0, clientPort=53564, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/cluster_97dbc57c-85a5-456c-2cc5-cce186832a72/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/cluster_97dbc57c-85a5-456c-2cc5-cce186832a72/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T17:58:07,827 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53564 2024-11-13T17:58:07,842 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T17:58:07,846 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T17:58:08,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741825_1001 (size=7) 2024-11-13T17:58:08,550 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0 with version=8 2024-11-13T17:58:08,550 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/hbase-staging 2024-11-13T17:58:08,656 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T17:58:09,027 INFO [Time-limited test {}] client.ConnectionUtils(128): master/39e84130bbc9:0 server-side Connection retries=9 2024-11-13T17:58:09,039 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T17:58:09,039 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T17:58:09,044 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T17:58:09,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T17:58:09,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T17:58:09,197 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T17:58:09,265 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T17:58:09,274 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T17:58:09,279 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T17:58:09,314 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1039 (auto-detected) 2024-11-13T17:58:09,318 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-13T17:58:09,351 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35773 2024-11-13T17:58:09,381 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35773 connecting to ZooKeeper ensemble=127.0.0.1:53564 2024-11-13T17:58:09,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:357730x0, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T17:58:09,441 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35773-0x100ed3fe3520000 connected 2024-11-13T17:58:09,576 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T17:58:09,580 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T17:58:09,614 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T17:58:09,621 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0, hbase.cluster.distributed=false 2024-11-13T17:58:09,676 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T17:58:09,686 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35773 2024-11-13T17:58:09,687 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35773 2024-11-13T17:58:09,689 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35773 2024-11-13T17:58:09,690 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35773 2024-11-13T17:58:09,693 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35773 2024-11-13T17:58:09,847 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/39e84130bbc9:0 server-side Connection retries=9 2024-11-13T17:58:09,849 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T17:58:09,850 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T17:58:09,851 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T17:58:09,851 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T17:58:09,851 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T17:58:09,855 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T17:58:09,857 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T17:58:09,858 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43723 2024-11-13T17:58:09,861 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43723 connecting to ZooKeeper ensemble=127.0.0.1:53564 2024-11-13T17:58:09,863 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T17:58:09,868 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T17:58:09,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:437230x0, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T17:58:09,878 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:437230x0, quorum=127.0.0.1:53564, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T17:58:09,881 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43723-0x100ed3fe3520001 connected 2024-11-13T17:58:09,885 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T17:58:09,897 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T17:58:09,901 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43723-0x100ed3fe3520001, quorum=127.0.0.1:53564, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T17:58:09,911 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43723-0x100ed3fe3520001, quorum=127.0.0.1:53564, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T17:58:09,917 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43723 2024-11-13T17:58:09,918 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43723 2024-11-13T17:58:09,919 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43723 2024-11-13T17:58:09,921 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43723 2024-11-13T17:58:09,924 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43723 2024-11-13T17:58:09,943 DEBUG [M:0;39e84130bbc9:35773 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;39e84130bbc9:35773 2024-11-13T17:58:09,944 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/39e84130bbc9,35773,1731520688728 2024-11-13T17:58:09,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43723-0x100ed3fe3520001, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T17:58:09,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T17:58:09,956 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/39e84130bbc9,35773,1731520688728 2024-11-13T17:58:10,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43723-0x100ed3fe3520001, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T17:58:10,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T17:58:10,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43723-0x100ed3fe3520001, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T17:58:10,009 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T17:58:10,013 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/39e84130bbc9,35773,1731520688728 from backup master directory 2024-11-13T17:58:10,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/39e84130bbc9,35773,1731520688728 2024-11-13T17:58:10,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43723-0x100ed3fe3520001, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T17:58:10,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T17:58:10,034 WARN [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T17:58:10,034 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=39e84130bbc9,35773,1731520688728 2024-11-13T17:58:10,039 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T17:58:10,041 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T17:58:10,127 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/hbase.id] with ID: f3eff791-1c75-435f-b397-9fb248642457 2024-11-13T17:58:10,127 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/.tmp/hbase.id 2024-11-13T17:58:10,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741826_1002 (size=42) 2024-11-13T17:58:10,151 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/.tmp/hbase.id]:[hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/hbase.id] 2024-11-13T17:58:10,226 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T17:58:10,231 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T17:58:10,275 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 38ms. 2024-11-13T17:58:10,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T17:58:10,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43723-0x100ed3fe3520001, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T17:58:10,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741827_1003 (size=196) 2024-11-13T17:58:10,375 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T17:58:10,389 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T17:58:10,413 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T17:58:10,418 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T17:58:10,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741828_1004 (size=1189) 2024-11-13T17:58:10,510 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/data/master/store 2024-11-13T17:58:10,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741829_1005 (size=34) 2024-11-13T17:58:10,540 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T17:58:10,543 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:58:10,545 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T17:58:10,545 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T17:58:10,545 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T17:58:10,546 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T17:58:10,547 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T17:58:10,547 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T17:58:10,549 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731520690545Disabling compacts and flushes for region at 1731520690545Disabling writes for close at 1731520690547 (+2 ms)Writing region close event to WAL at 1731520690547Closed at 1731520690547 2024-11-13T17:58:10,552 WARN [master/39e84130bbc9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/data/master/store/.initializing 2024-11-13T17:58:10,552 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/WALs/39e84130bbc9,35773,1731520688728 2024-11-13T17:58:10,564 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T17:58:10,583 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=39e84130bbc9%2C35773%2C1731520688728, suffix=, logDir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/WALs/39e84130bbc9,35773,1731520688728, archiveDir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/oldWALs, maxLogs=10 2024-11-13T17:58:10,619 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/WALs/39e84130bbc9,35773,1731520688728/39e84130bbc9%2C35773%2C1731520688728.1731520690589, exclude list is [], retry=0 2024-11-13T17:58:10,641 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39637,DS-5f126945-7071-47ac-9261-62e668c3d3ed,DISK] 2024-11-13T17:58:10,644 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T17:58:10,693 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/WALs/39e84130bbc9,35773,1731520688728/39e84130bbc9%2C35773%2C1731520688728.1731520690589 2024-11-13T17:58:10,694 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33155:33155)] 2024-11-13T17:58:10,695 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T17:58:10,696 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:58:10,700 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T17:58:10,701 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T17:58:10,762 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T17:58:10,797 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T17:58:10,802 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:58:10,805 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T17:58:10,806 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T17:58:10,810 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T17:58:10,811 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:58:10,812 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T17:58:10,812 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T17:58:10,816 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T17:58:10,816 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:58:10,817 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T17:58:10,817 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T17:58:10,823 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T17:58:10,823 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:58:10,825 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T17:58:10,825 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T17:58:10,830 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T17:58:10,831 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T17:58:10,838 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T17:58:10,839 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T17:58:10,844 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T17:58:10,849 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T17:58:10,857 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T17:58:10,859 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71160189, jitterRate=0.060369446873664856}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T17:58:10,865 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731520690721Initializing all the Stores at 1731520690724 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731520690725 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731520690726 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731520690727 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731520690727Cleaning up temporary data from old regions at 1731520690839 (+112 ms)Region opened successfully at 1731520690865 (+26 ms) 2024-11-13T17:58:10,867 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T17:58:10,911 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74c77412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=39e84130bbc9/172.17.0.3:0 2024-11-13T17:58:10,962 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T17:58:10,984 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T17:58:10,985 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T17:58:10,989 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T17:58:10,994 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 4 msec 2024-11-13T17:58:11,005 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-13T17:58:11,005 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T17:58:11,054 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T17:58:11,066 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T17:58:11,073 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T17:58:11,077 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T17:58:11,079 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T17:58:11,082 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T17:58:11,085 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T17:58:11,090 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T17:58:11,092 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T17:58:11,094 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T17:58:11,099 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T17:58:11,123 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T17:58:11,125 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T17:58:11,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T17:58:11,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43723-0x100ed3fe3520001, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T17:58:11,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T17:58:11,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43723-0x100ed3fe3520001, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T17:58:11,134 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=39e84130bbc9,35773,1731520688728, sessionid=0x100ed3fe3520000, setting cluster-up flag (Was=false) 2024-11-13T17:58:11,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43723-0x100ed3fe3520001, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T17:58:11,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T17:58:11,163 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T17:58:11,165 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=39e84130bbc9,35773,1731520688728 2024-11-13T17:58:11,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T17:58:11,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43723-0x100ed3fe3520001, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T17:58:11,183 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T17:58:11,187 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=39e84130bbc9,35773,1731520688728 2024-11-13T17:58:11,194 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T17:58:11,236 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(746): ClusterId : f3eff791-1c75-435f-b397-9fb248642457 2024-11-13T17:58:11,238 DEBUG [RS:0;39e84130bbc9:43723 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T17:58:11,247 DEBUG [RS:0;39e84130bbc9:43723 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T17:58:11,247 DEBUG [RS:0;39e84130bbc9:43723 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T17:58:11,251 DEBUG [RS:0;39e84130bbc9:43723 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T17:58:11,251 DEBUG [RS:0;39e84130bbc9:43723 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@224ab3f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=39e84130bbc9/172.17.0.3:0 2024-11-13T17:58:11,269 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;39e84130bbc9:43723 2024-11-13T17:58:11,273 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T17:58:11,273 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T17:58:11,273 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T17:58:11,276 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(2659): reportForDuty to master=39e84130bbc9,35773,1731520688728 with port=43723, startcode=1731520689798 2024-11-13T17:58:11,279 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T17:58:11,288 DEBUG [RS:0;39e84130bbc9:43723 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T17:58:11,293 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T17:58:11,303 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T17:58:11,310 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 39e84130bbc9,35773,1731520688728 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T17:58:11,332 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/39e84130bbc9:0, corePoolSize=5, maxPoolSize=5 2024-11-13T17:58:11,333 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/39e84130bbc9:0, corePoolSize=5, maxPoolSize=5 2024-11-13T17:58:11,333 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/39e84130bbc9:0, corePoolSize=5, maxPoolSize=5 2024-11-13T17:58:11,333 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/39e84130bbc9:0, corePoolSize=5, maxPoolSize=5 2024-11-13T17:58:11,334 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/39e84130bbc9:0, corePoolSize=10, maxPoolSize=10 2024-11-13T17:58:11,336 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T17:58:11,336 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/39e84130bbc9:0, corePoolSize=2, maxPoolSize=2 2024-11-13T17:58:11,337 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T17:58:11,340 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43529, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T17:58:11,347 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T17:58:11,348 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T17:58:11,354 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731520721354 2024-11-13T17:58:11,356 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T17:58:11,358 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T17:58:11,359 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35773 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-13T17:58:11,376 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T17:58:11,376 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T17:58:11,377 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T17:58:11,377 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T17:58:11,379 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:58:11,380 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T17:58:11,384 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T17:58:11,397 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T17:58:11,398 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T17:58:11,399 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T17:58:11,403 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T17:58:11,403 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T17:58:11,406 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/39e84130bbc9:0:becomeActiveMaster-HFileCleaner.large.0-1731520691405,5,FailOnTimeoutGroup] 2024-11-13T17:58:11,411 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T17:58:11,411 WARN [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 30000 ms and then retrying. 2024-11-13T17:58:11,413 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/39e84130bbc9:0:becomeActiveMaster-HFileCleaner.small.0-1731520691406,5,FailOnTimeoutGroup] 2024-11-13T17:58:11,414 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T17:58:11,414 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T17:58:11,415 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T17:58:11,416 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T17:58:11,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741831_1007 (size=1321) 2024-11-13T17:58:11,821 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T17:58:11,822 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0 2024-11-13T17:58:11,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741832_1008 (size=32) 2024-11-13T17:58:11,845 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:58:11,852 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T17:58:11,863 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T17:58:11,863 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:58:11,864 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T17:58:11,865 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T17:58:11,869 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T17:58:11,869 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:58:11,870 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T17:58:11,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T17:58:11,874 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T17:58:11,874 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:58:11,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T17:58:11,876 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T17:58:11,879 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T17:58:11,879 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:58:11,880 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T17:58:11,881 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T17:58:11,882 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740 2024-11-13T17:58:11,883 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740 2024-11-13T17:58:11,889 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T17:58:11,889 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T17:58:11,890 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T17:58:11,898 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T17:58:11,906 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T17:58:11,908 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71540978, jitterRate=0.06604364514350891}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T17:58:11,912 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731520691845Initializing all the Stores at 1731520691851 (+6 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731520691851Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731520691852 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731520691852Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731520691852Cleaning up temporary data from old regions at 1731520691889 (+37 ms)Region opened successfully at 1731520691912 (+23 ms) 2024-11-13T17:58:11,912 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T17:58:11,913 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T17:58:11,913 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T17:58:11,913 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T17:58:11,913 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T17:58:11,915 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T17:58:11,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731520691912Disabling compacts and flushes for region at 1731520691912Disabling writes for close at 1731520691913 (+1 ms)Writing region close event to WAL at 1731520691914 (+1 ms)Closed at 1731520691915 (+1 ms) 2024-11-13T17:58:11,920 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T17:58:11,920 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T17:58:11,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T17:58:11,944 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T17:58:11,949 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T17:58:12,100 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:13,101 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:14,102 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:15,102 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:16,103 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:17,104 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:18,105 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:19,105 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:19,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T17:58:19,261 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T17:58:20,106 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:21,107 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:22,108 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:23,108 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:24,109 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:25,110 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:26,111 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:27,111 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:28,112 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:29,113 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:30,114 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:31,114 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:32,115 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:33,116 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:34,117 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:35,117 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:36,118 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:37,119 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:37,671 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T17:58:38,120 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:39,121 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:40,121 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:41,122 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:41,413 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(2659): reportForDuty to master=39e84130bbc9,35773,1731520688728 with port=43723, startcode=1731520689798 2024-11-13T17:58:41,416 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35773 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 39e84130bbc9,43723,1731520689798 2024-11-13T17:58:41,420 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35773 {}] master.ServerManager(517): Registering regionserver=39e84130bbc9,43723,1731520689798 2024-11-13T17:58:41,434 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0 2024-11-13T17:58:41,434 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33935 2024-11-13T17:58:41,434 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T17:58:41,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T17:58:41,441 DEBUG [RS:0;39e84130bbc9:43723 {}] zookeeper.ZKUtil(111): regionserver:43723-0x100ed3fe3520001, quorum=127.0.0.1:53564, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/39e84130bbc9,43723,1731520689798 2024-11-13T17:58:41,441 WARN [RS:0;39e84130bbc9:43723 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T17:58:41,441 INFO [RS:0;39e84130bbc9:43723 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T17:58:41,442 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/WALs/39e84130bbc9,43723,1731520689798 2024-11-13T17:58:41,446 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [39e84130bbc9,43723,1731520689798] 2024-11-13T17:58:41,512 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T17:58:41,531 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T17:58:41,541 INFO [RS:0;39e84130bbc9:43723 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T17:58:41,541 INFO [RS:0;39e84130bbc9:43723 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T17:58:41,547 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T17:58:41,556 INFO [RS:0;39e84130bbc9:43723 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T17:58:41,560 INFO [RS:0;39e84130bbc9:43723 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T17:58:41,560 DEBUG [RS:0;39e84130bbc9:43723 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T17:58:41,560 DEBUG [RS:0;39e84130bbc9:43723 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T17:58:41,561 DEBUG [RS:0;39e84130bbc9:43723 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T17:58:41,561 DEBUG [RS:0;39e84130bbc9:43723 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T17:58:41,561 DEBUG [RS:0;39e84130bbc9:43723 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T17:58:41,561 DEBUG [RS:0;39e84130bbc9:43723 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/39e84130bbc9:0, corePoolSize=2, maxPoolSize=2 2024-11-13T17:58:41,562 DEBUG [RS:0;39e84130bbc9:43723 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T17:58:41,562 DEBUG [RS:0;39e84130bbc9:43723 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T17:58:41,562 DEBUG [RS:0;39e84130bbc9:43723 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T17:58:41,562 DEBUG [RS:0;39e84130bbc9:43723 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T17:58:41,562 DEBUG [RS:0;39e84130bbc9:43723 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T17:58:41,563 DEBUG [RS:0;39e84130bbc9:43723 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/39e84130bbc9:0, corePoolSize=1, maxPoolSize=1 2024-11-13T17:58:41,563 DEBUG [RS:0;39e84130bbc9:43723 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/39e84130bbc9:0, corePoolSize=3, maxPoolSize=3 2024-11-13T17:58:41,563 DEBUG [RS:0;39e84130bbc9:43723 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/39e84130bbc9:0, corePoolSize=3, maxPoolSize=3 2024-11-13T17:58:41,566 INFO [RS:0;39e84130bbc9:43723 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T17:58:41,566 INFO [RS:0;39e84130bbc9:43723 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T17:58:41,567 INFO [RS:0;39e84130bbc9:43723 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T17:58:41,567 INFO [RS:0;39e84130bbc9:43723 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T17:58:41,567 INFO [RS:0;39e84130bbc9:43723 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T17:58:41,567 INFO [RS:0;39e84130bbc9:43723 {}] hbase.ChoreService(168): Chore ScheduledChore name=39e84130bbc9,43723,1731520689798-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T17:58:41,606 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T17:58:41,609 INFO [RS:0;39e84130bbc9:43723 {}] hbase.ChoreService(168): Chore ScheduledChore name=39e84130bbc9,43723,1731520689798-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T17:58:41,610 INFO [RS:0;39e84130bbc9:43723 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T17:58:41,610 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.Replication(171): 39e84130bbc9,43723,1731520689798 started 2024-11-13T17:58:41,644 INFO [RS:0;39e84130bbc9:43723 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T17:58:41,644 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1482): Serving as 39e84130bbc9,43723,1731520689798, RpcServer on 39e84130bbc9/172.17.0.3:43723, sessionid=0x100ed3fe3520001 2024-11-13T17:58:41,649 DEBUG [RS:0;39e84130bbc9:43723 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T17:58:41,650 DEBUG [RS:0;39e84130bbc9:43723 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 39e84130bbc9,43723,1731520689798 2024-11-13T17:58:41,653 DEBUG [RS:0;39e84130bbc9:43723 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '39e84130bbc9,43723,1731520689798' 2024-11-13T17:58:41,654 DEBUG [RS:0;39e84130bbc9:43723 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T17:58:41,655 DEBUG [RS:0;39e84130bbc9:43723 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T17:58:41,659 DEBUG [RS:0;39e84130bbc9:43723 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T17:58:41,659 DEBUG [RS:0;39e84130bbc9:43723 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T17:58:41,659 DEBUG [RS:0;39e84130bbc9:43723 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 39e84130bbc9,43723,1731520689798 2024-11-13T17:58:41,659 DEBUG [RS:0;39e84130bbc9:43723 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '39e84130bbc9,43723,1731520689798' 2024-11-13T17:58:41,659 DEBUG [RS:0;39e84130bbc9:43723 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T17:58:41,660 DEBUG [RS:0;39e84130bbc9:43723 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T17:58:41,665 DEBUG [RS:0;39e84130bbc9:43723 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T17:58:41,666 INFO [RS:0;39e84130bbc9:43723 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T17:58:41,666 INFO [RS:0;39e84130bbc9:43723 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T17:58:42,123 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:43,124 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:44,124 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:45,125 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:46,126 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:47,126 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:48,127 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:49,128 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:50,129 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:51,129 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:52,130 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:53,131 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:54,132 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:55,132 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:56,133 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:57,134 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:58,135 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:58:59,135 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:59:00,136 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:59:01,137 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:59:02,137 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:59:03,138 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:59:04,139 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:59:05,140 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:59:06,140 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:59:07,141 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:59:07,671 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T17:59:08,142 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:59:09,142 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:59:10,143 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:59:11,144 WARN [39e84130bbc9:35773 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T17:59:11,671 INFO [RS:0;39e84130bbc9:43723 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T17:59:11,676 INFO [RS:0;39e84130bbc9:43723 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=39e84130bbc9%2C43723%2C1731520689798, suffix=, logDir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/WALs/39e84130bbc9,43723,1731520689798, archiveDir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/oldWALs, maxLogs=32 2024-11-13T17:59:11,693 DEBUG [RS:0;39e84130bbc9:43723 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/WALs/39e84130bbc9,43723,1731520689798/39e84130bbc9%2C43723%2C1731520689798.1731520751678, exclude list is [], retry=0 2024-11-13T17:59:11,698 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39637,DS-5f126945-7071-47ac-9261-62e668c3d3ed,DISK] 2024-11-13T17:59:11,703 INFO [RS:0;39e84130bbc9:43723 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/WALs/39e84130bbc9,43723,1731520689798/39e84130bbc9%2C43723%2C1731520689798.1731520751678 2024-11-13T17:59:11,704 DEBUG [RS:0;39e84130bbc9:43723 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33155:33155)] 2024-11-13T17:59:11,896 DEBUG [39e84130bbc9:35773 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T17:59:11,909 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:11,916 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39e84130bbc9,43723,1731520689798, state=OPENING 2024-11-13T17:59:11,922 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T17:59:11,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43723-0x100ed3fe3520001, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T17:59:11,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T17:59:11,925 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T17:59:11,926 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T17:59:11,927 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T17:59:11,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=39e84130bbc9,43723,1731520689798}] 2024-11-13T17:59:12,105 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T17:59:12,109 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42387, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T17:59:12,120 INFO [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T17:59:12,120 INFO [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T17:59:12,120 INFO [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T17:59:12,123 INFO [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=39e84130bbc9%2C43723%2C1731520689798.meta, suffix=.meta, logDir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/WALs/39e84130bbc9,43723,1731520689798, archiveDir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/oldWALs, maxLogs=32 2024-11-13T17:59:12,142 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/WALs/39e84130bbc9,43723,1731520689798/39e84130bbc9%2C43723%2C1731520689798.meta.1731520752125.meta, exclude list is [], retry=0 2024-11-13T17:59:12,146 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39637,DS-5f126945-7071-47ac-9261-62e668c3d3ed,DISK] 2024-11-13T17:59:12,149 INFO [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/WALs/39e84130bbc9,43723,1731520689798/39e84130bbc9%2C43723%2C1731520689798.meta.1731520752125.meta 2024-11-13T17:59:12,150 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33155:33155)] 2024-11-13T17:59:12,150 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T17:59:12,152 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T17:59:12,154 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T17:59:12,159 INFO [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T17:59:12,163 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T17:59:12,164 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:59:12,164 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T17:59:12,164 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T17:59:12,167 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T17:59:12,169 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T17:59:12,169 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:12,170 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T17:59:12,170 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T17:59:12,171 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T17:59:12,171 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:12,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T17:59:12,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T17:59:12,174 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T17:59:12,174 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:12,175 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T17:59:12,175 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T17:59:12,176 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T17:59:12,176 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:12,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T17:59:12,177 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T17:59:12,179 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740 2024-11-13T17:59:12,181 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740 2024-11-13T17:59:12,183 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T17:59:12,184 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T17:59:12,184 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T17:59:12,187 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T17:59:12,189 INFO [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72582168, jitterRate=0.08155858516693115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T17:59:12,189 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T17:59:12,190 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731520752165Writing region info on filesystem at 1731520752165Initializing all the Stores at 1731520752167 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731520752167Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731520752167Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731520752167Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731520752167Cleaning up temporary data from old regions at 1731520752184 (+17 ms)Running coprocessor post-open hooks at 1731520752189 (+5 ms)Region opened successfully at 1731520752190 (+1 ms) 2024-11-13T17:59:12,198 INFO [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731520752096 2024-11-13T17:59:12,210 DEBUG [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T17:59:12,211 INFO [RS_OPEN_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T17:59:12,213 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:12,215 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39e84130bbc9,43723,1731520689798, state=OPEN 2024-11-13T17:59:12,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T17:59:12,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43723-0x100ed3fe3520001, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T17:59:12,221 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T17:59:12,221 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T17:59:12,221 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:12,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T17:59:12,227 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39e84130bbc9,43723,1731520689798 in 293 msec 2024-11-13T17:59:12,233 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T17:59:12,233 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1 mins, 0.299 sec 2024-11-13T17:59:12,234 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T17:59:12,234 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T17:59:12,258 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T17:59:12,260 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,43723,1731520689798, seqNum=-1] 2024-11-13T17:59:12,291 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T17:59:12,297 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56033, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T17:59:12,330 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1 mins, 1.102 sec 2024-11-13T17:59:12,331 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731520752330, completionTime=-1 2024-11-13T17:59:12,335 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-13T17:59:12,335 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T17:59:12,370 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-13T17:59:12,370 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731520812370 2024-11-13T17:59:12,370 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731520872370 2024-11-13T17:59:12,370 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 35 msec 2024-11-13T17:59:12,372 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39e84130bbc9,35773,1731520688728-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T17:59:12,373 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39e84130bbc9,35773,1731520688728-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T17:59:12,373 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39e84130bbc9,35773,1731520688728-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T17:59:12,374 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-39e84130bbc9:35773, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T17:59:12,375 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T17:59:12,376 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T17:59:12,383 DEBUG [master/39e84130bbc9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T17:59:12,404 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 62.370sec 2024-11-13T17:59:12,405 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T17:59:12,406 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T17:59:12,407 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T17:59:12,408 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T17:59:12,408 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T17:59:12,409 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39e84130bbc9,35773,1731520688728-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T17:59:12,409 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39e84130bbc9,35773,1731520688728-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T17:59:12,418 DEBUG [master/39e84130bbc9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T17:59:12,419 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T17:59:12,419 INFO [master/39e84130bbc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39e84130bbc9,35773,1731520688728-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T17:59:12,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2918f47c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T17:59:12,503 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T17:59:12,503 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T17:59:12,507 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,35773,-1 for getting cluster id 2024-11-13T17:59:12,510 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T17:59:12,521 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f3eff791-1c75-435f-b397-9fb248642457' 2024-11-13T17:59:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T17:59:12,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f3eff791-1c75-435f-b397-9fb248642457" 2024-11-13T17:59:12,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fa6722b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T17:59:12,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,35773,-1] 2024-11-13T17:59:12,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T17:59:12,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T17:59:12,542 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T17:59:12,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c8b08a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T17:59:12,546 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T17:59:12,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,43723,1731520689798, seqNum=-1] 2024-11-13T17:59:12,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T17:59:12,558 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T17:59:12,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=39e84130bbc9,35773,1731520688728 2024-11-13T17:59:12,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@438beceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T17:59:12,582 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39e84130bbc9,35773,-1 for getting cluster id 2024-11-13T17:59:12,582 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T17:59:12,584 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f3eff791-1c75-435f-b397-9fb248642457' 2024-11-13T17:59:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T17:59:12,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f3eff791-1c75-435f-b397-9fb248642457" 2024-11-13T17:59:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@569d676e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T17:59:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39e84130bbc9,35773,-1] 2024-11-13T17:59:12,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T17:59:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T17:59:12,589 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T17:59:12,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3170ebff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T17:59:12,626 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=208, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=485, ProcessCount=11, AvailableMemoryMB=2805 2024-11-13T17:59:12,635 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T17:59:12,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 39e84130bbc9,35773,1731520688728 2024-11-13T17:59:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@c0f74fa 2024-11-13T17:59:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T17:59:12,678 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T17:59:12,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T17:59:12,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-13T17:59:12,711 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T17:59:12,715 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:12,722 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T17:59:12,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-13T17:59:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T17:59:12,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741835_1011 (size=428) 2024-11-13T17:59:12,769 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 589c6b5058af0a6bb92e99c352ec34b9, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0 2024-11-13T17:59:12,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741836_1012 (size=79) 2024-11-13T17:59:12,804 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:59:12,804 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 589c6b5058af0a6bb92e99c352ec34b9, disabling compactions & flushes 2024-11-13T17:59:12,804 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9. 2024-11-13T17:59:12,804 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9. 2024-11-13T17:59:12,804 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9. after waiting 0 ms 2024-11-13T17:59:12,804 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9. 2024-11-13T17:59:12,804 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9. 2024-11-13T17:59:12,804 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 589c6b5058af0a6bb92e99c352ec34b9: Waiting for close lock at 1731520752804Disabling compacts and flushes for region at 1731520752804Disabling writes for close at 1731520752804Writing region close event to WAL at 1731520752804Closed at 1731520752804 2024-11-13T17:59:12,812 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T17:59:12,818 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731520752812"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731520752812"}]},"ts":"1731520752812"} 2024-11-13T17:59:12,831 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T17:59:12,834 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T17:59:12,840 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731520752835"}]},"ts":"1731520752835"} 2024-11-13T17:59:12,851 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-13T17:59:12,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=589c6b5058af0a6bb92e99c352ec34b9, ASSIGN}] 2024-11-13T17:59:12,863 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=589c6b5058af0a6bb92e99c352ec34b9, ASSIGN 2024-11-13T17:59:12,874 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=589c6b5058af0a6bb92e99c352ec34b9, ASSIGN; state=OFFLINE, location=39e84130bbc9,43723,1731520689798; forceNewPlan=false, retain=false 2024-11-13T17:59:13,028 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=589c6b5058af0a6bb92e99c352ec34b9, regionState=OPENING, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:13,034 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=589c6b5058af0a6bb92e99c352ec34b9, ASSIGN because future has completed 2024-11-13T17:59:13,035 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 589c6b5058af0a6bb92e99c352ec34b9, server=39e84130bbc9,43723,1731520689798}] 2024-11-13T17:59:13,197 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9. 2024-11-13T17:59:13,198 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 589c6b5058af0a6bb92e99c352ec34b9, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9.', STARTKEY => '', ENDKEY => ''} 2024-11-13T17:59:13,198 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 589c6b5058af0a6bb92e99c352ec34b9 2024-11-13T17:59:13,199 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:59:13,199 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 589c6b5058af0a6bb92e99c352ec34b9 2024-11-13T17:59:13,199 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 589c6b5058af0a6bb92e99c352ec34b9 2024-11-13T17:59:13,202 INFO [StoreOpener-589c6b5058af0a6bb92e99c352ec34b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 589c6b5058af0a6bb92e99c352ec34b9 2024-11-13T17:59:13,204 INFO [StoreOpener-589c6b5058af0a6bb92e99c352ec34b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 589c6b5058af0a6bb92e99c352ec34b9 columnFamilyName testFamily 2024-11-13T17:59:13,204 DEBUG [StoreOpener-589c6b5058af0a6bb92e99c352ec34b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:13,205 INFO [StoreOpener-589c6b5058af0a6bb92e99c352ec34b9-1 {}] regionserver.HStore(327): Store=589c6b5058af0a6bb92e99c352ec34b9/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T17:59:13,206 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 589c6b5058af0a6bb92e99c352ec34b9 2024-11-13T17:59:13,207 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testRetryOutOfOrderScannerNextExceptionAsync/589c6b5058af0a6bb92e99c352ec34b9 2024-11-13T17:59:13,208 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testRetryOutOfOrderScannerNextExceptionAsync/589c6b5058af0a6bb92e99c352ec34b9 2024-11-13T17:59:13,208 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 589c6b5058af0a6bb92e99c352ec34b9 2024-11-13T17:59:13,208 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 589c6b5058af0a6bb92e99c352ec34b9 2024-11-13T17:59:13,211 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 589c6b5058af0a6bb92e99c352ec34b9 2024-11-13T17:59:13,215 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testRetryOutOfOrderScannerNextExceptionAsync/589c6b5058af0a6bb92e99c352ec34b9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T17:59:13,216 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 589c6b5058af0a6bb92e99c352ec34b9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70364378, jitterRate=0.04851093888282776}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T17:59:13,216 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 589c6b5058af0a6bb92e99c352ec34b9 2024-11-13T17:59:13,217 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 589c6b5058af0a6bb92e99c352ec34b9: Running coprocessor pre-open hook at 1731520753199Writing region info on filesystem at 1731520753199Initializing all the Stores at 1731520753201 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731520753201Cleaning up temporary data from old regions at 1731520753208 (+7 ms)Running coprocessor post-open hooks at 1731520753216 (+8 ms)Region opened successfully at 1731520753217 (+1 ms) 2024-11-13T17:59:13,220 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9., pid=6, masterSystemTime=1731520753190 2024-11-13T17:59:13,224 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9. 2024-11-13T17:59:13,224 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9. 2024-11-13T17:59:13,225 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=589c6b5058af0a6bb92e99c352ec34b9, regionState=OPEN, openSeqNum=2, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:13,230 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 589c6b5058af0a6bb92e99c352ec34b9, server=39e84130bbc9,43723,1731520689798 because future has completed 2024-11-13T17:59:13,240 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T17:59:13,241 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 589c6b5058af0a6bb92e99c352ec34b9, server=39e84130bbc9,43723,1731520689798 in 200 msec 2024-11-13T17:59:13,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T17:59:13,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=589c6b5058af0a6bb92e99c352ec34b9, ASSIGN in 386 msec 2024-11-13T17:59:13,245 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T17:59:13,246 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731520753246"}]},"ts":"1731520753246"} 2024-11-13T17:59:13,249 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-13T17:59:13,251 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T17:59:13,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 563 msec 2024-11-13T17:59:13,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T17:59:13,763 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-13T17:59:13,763 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-13T17:59:13,764 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T17:59:13,769 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-13T17:59:13,770 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T17:59:13,770 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-13T17:59:13,773 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T17:59:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39e84130bbc9,43723,1731520689798, seqNum=-1] 2024-11-13T17:59:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T17:59:13,779 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T17:59:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9., hostname=39e84130bbc9,43723,1731520689798, seqNum=2] 2024-11-13T17:59:13,823 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-13T17:59:13,829 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-13T17:59:13,838 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-13T17:59:13,846 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-13T17:59:13,848 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-13T17:59:13,848 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=3409586232043241480 2024-11-13T17:59:13,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:47138 deadline: 1731520756847, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-13T17:59:13,859 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-13T17:59:13,861 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-13T17:59:13,862 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-13T17:59:13,864 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-13T17:59:13,865 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=3409586232043241482 2024-11-13T17:59:13,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:47138 deadline: 1731520756863, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-13T17:59:13,869 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-13T17:59:13,870 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=3409586232043241483 2024-11-13T17:59:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:47138 deadline: 1731520756869, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-13T17:59:13,874 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-13T17:59:13,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=3409586232043241484 2024-11-13T17:59:13,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:47138 deadline: 1731520756874, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-13T17:59:13,879 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-13T17:59:13,891 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=215 (was 208) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1042495474_22 at /127.0.0.1:52236 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=526 (was 485) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2724 (was 2805) 2024-11-13T17:59:13,900 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=215, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=526, ProcessCount=11, AvailableMemoryMB=2724 2024-11-13T17:59:13,903 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35773 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T17:59:13,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35773 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-13T17:59:13,907 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T17:59:13,907 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:13,908 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35773 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-13T17:59:13,909 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T17:59:13,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T17:59:13,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741837_1013 (size=414) 2024-11-13T17:59:13,920 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0a98fe002b6f18a2d07205dfbffd3b04, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0 2024-11-13T17:59:13,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741838_1014 (size=65) 2024-11-13T17:59:14,329 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:59:14,329 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 0a98fe002b6f18a2d07205dfbffd3b04, disabling compactions & flushes 2024-11-13T17:59:14,329 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04. 2024-11-13T17:59:14,330 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04. 2024-11-13T17:59:14,330 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04. after waiting 0 ms 2024-11-13T17:59:14,330 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04. 2024-11-13T17:59:14,330 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04. 2024-11-13T17:59:14,330 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0a98fe002b6f18a2d07205dfbffd3b04: Waiting for close lock at 1731520754329Disabling compacts and flushes for region at 1731520754329Disabling writes for close at 1731520754330 (+1 ms)Writing region close event to WAL at 1731520754330Closed at 1731520754330 2024-11-13T17:59:14,332 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T17:59:14,332 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1731520754332"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731520754332"}]},"ts":"1731520754332"} 2024-11-13T17:59:14,336 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T17:59:14,337 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T17:59:14,338 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731520754337"}]},"ts":"1731520754337"} 2024-11-13T17:59:14,341 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-13T17:59:14,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0a98fe002b6f18a2d07205dfbffd3b04, ASSIGN}] 2024-11-13T17:59:14,343 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0a98fe002b6f18a2d07205dfbffd3b04, ASSIGN 2024-11-13T17:59:14,345 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0a98fe002b6f18a2d07205dfbffd3b04, ASSIGN; state=OFFLINE, location=39e84130bbc9,43723,1731520689798; forceNewPlan=false, retain=false 2024-11-13T17:59:14,496 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=0a98fe002b6f18a2d07205dfbffd3b04, regionState=OPENING, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:14,501 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0a98fe002b6f18a2d07205dfbffd3b04, ASSIGN because future has completed 2024-11-13T17:59:14,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a98fe002b6f18a2d07205dfbffd3b04, server=39e84130bbc9,43723,1731520689798}] 2024-11-13T17:59:14,662 INFO [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04. 2024-11-13T17:59:14,662 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 0a98fe002b6f18a2d07205dfbffd3b04, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04.', STARTKEY => '', ENDKEY => ''} 2024-11-13T17:59:14,662 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 0a98fe002b6f18a2d07205dfbffd3b04 2024-11-13T17:59:14,663 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:59:14,663 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 0a98fe002b6f18a2d07205dfbffd3b04 2024-11-13T17:59:14,663 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 0a98fe002b6f18a2d07205dfbffd3b04 2024-11-13T17:59:14,665 INFO [StoreOpener-0a98fe002b6f18a2d07205dfbffd3b04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0a98fe002b6f18a2d07205dfbffd3b04 2024-11-13T17:59:14,667 INFO [StoreOpener-0a98fe002b6f18a2d07205dfbffd3b04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a98fe002b6f18a2d07205dfbffd3b04 columnFamilyName testFamily 2024-11-13T17:59:14,667 DEBUG [StoreOpener-0a98fe002b6f18a2d07205dfbffd3b04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:14,668 INFO [StoreOpener-0a98fe002b6f18a2d07205dfbffd3b04-1 {}] regionserver.HStore(327): Store=0a98fe002b6f18a2d07205dfbffd3b04/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T17:59:14,668 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 0a98fe002b6f18a2d07205dfbffd3b04 2024-11-13T17:59:14,669 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnOpenScanner/0a98fe002b6f18a2d07205dfbffd3b04 2024-11-13T17:59:14,670 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnOpenScanner/0a98fe002b6f18a2d07205dfbffd3b04 2024-11-13T17:59:14,671 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 0a98fe002b6f18a2d07205dfbffd3b04 2024-11-13T17:59:14,671 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 0a98fe002b6f18a2d07205dfbffd3b04 2024-11-13T17:59:14,673 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 0a98fe002b6f18a2d07205dfbffd3b04 2024-11-13T17:59:14,676 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnOpenScanner/0a98fe002b6f18a2d07205dfbffd3b04/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T17:59:14,677 INFO [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 0a98fe002b6f18a2d07205dfbffd3b04; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65371501, jitterRate=-0.025888726115226746}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T17:59:14,677 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0a98fe002b6f18a2d07205dfbffd3b04 2024-11-13T17:59:14,678 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 0a98fe002b6f18a2d07205dfbffd3b04: Running coprocessor pre-open hook at 1731520754663Writing region info on filesystem at 1731520754663Initializing all the Stores at 1731520754664 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731520754665 (+1 ms)Cleaning up temporary data from old regions at 1731520754671 (+6 ms)Running coprocessor post-open hooks at 1731520754677 (+6 ms)Region opened successfully at 1731520754678 (+1 ms) 2024-11-13T17:59:14,679 INFO [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04., pid=9, masterSystemTime=1731520754656 2024-11-13T17:59:14,683 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04. 2024-11-13T17:59:14,683 INFO [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04. 2024-11-13T17:59:14,684 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=0a98fe002b6f18a2d07205dfbffd3b04, regionState=OPEN, openSeqNum=2, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:14,687 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a98fe002b6f18a2d07205dfbffd3b04, server=39e84130bbc9,43723,1731520689798 because future has completed 2024-11-13T17:59:14,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-13T17:59:14,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 0a98fe002b6f18a2d07205dfbffd3b04, server=39e84130bbc9,43723,1731520689798 in 187 msec 2024-11-13T17:59:14,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T17:59:14,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=0a98fe002b6f18a2d07205dfbffd3b04, ASSIGN in 351 msec 2024-11-13T17:59:14,697 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T17:59:14,697 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731520754697"}]},"ts":"1731520754697"} 2024-11-13T17:59:14,700 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-13T17:59:14,701 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T17:59:14,704 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 797 msec 2024-11-13T17:59:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T17:59:14,921 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-13T17:59:14,921 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-13T17:59:14,922 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T17:59:14,926 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-13T17:59:14,926 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T17:59:14,926 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-13T17:59:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04., hostname=39e84130bbc9,43723,1731520689798, seqNum=2] 2024-11-13T17:59:14,942 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-13T17:59:14,942 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-13T17:59:14,946 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-13T17:59:14,947 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43723 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-13T17:59:17,706 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T17:59:17,738 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-13T17:59:17,740 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T17:59:17,741 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-13T17:59:19,259 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T17:59:19,259 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T17:59:19,261 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T17:59:19,261 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T17:59:19,261 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-13T17:59:19,261 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-13T17:59:19,262 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-13T17:59:19,262 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-13T17:59:20,961 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04., hostname=39e84130bbc9,43723,1731520689798, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04., hostname=39e84130bbc9,43723,1731520689798, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=39e84130bbc9:43723 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6012ms, rpcTimeout=5999ms 2024-11-13T17:59:20,961 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04., hostname=39e84130bbc9,43723,1731520689798, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6012ms, rpcTimeout=5999ms 2024-11-13T17:59:20,962 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04., hostname=39e84130bbc9,43723,1731520689798, seqNum=2 from cache 2024-11-13T17:59:20,976 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=218 (was 215) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=508 (was 526), ProcessCount=11 (was 11), AvailableMemoryMB=2405 (was 2724) 2024-11-13T17:59:20,986 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=218, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=508, ProcessCount=11, AvailableMemoryMB=2404 2024-11-13T17:59:20,990 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35773 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T17:59:20,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35773 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-13T17:59:20,994 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T17:59:20,994 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:20,994 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35773 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-13T17:59:20,996 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T17:59:20,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-13T17:59:21,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741839_1015 (size=412) 2024-11-13T17:59:21,414 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dc926cd9c7c1f78f2949ef7a52bf7b2d, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0 2024-11-13T17:59:21,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741840_1016 (size=63) 2024-11-13T17:59:21,428 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:59:21,428 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing dc926cd9c7c1f78f2949ef7a52bf7b2d, disabling compactions & flushes 2024-11-13T17:59:21,428 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d. 2024-11-13T17:59:21,428 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d. 2024-11-13T17:59:21,428 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d. after waiting 0 ms 2024-11-13T17:59:21,428 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d. 2024-11-13T17:59:21,428 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d. 2024-11-13T17:59:21,429 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for dc926cd9c7c1f78f2949ef7a52bf7b2d: Waiting for close lock at 1731520761428Disabling compacts and flushes for region at 1731520761428Disabling writes for close at 1731520761428Writing region close event to WAL at 1731520761428Closed at 1731520761428 2024-11-13T17:59:21,431 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T17:59:21,431 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1731520761431"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731520761431"}]},"ts":"1731520761431"} 2024-11-13T17:59:21,435 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T17:59:21,436 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T17:59:21,437 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731520761436"}]},"ts":"1731520761436"} 2024-11-13T17:59:21,440 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-13T17:59:21,440 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=dc926cd9c7c1f78f2949ef7a52bf7b2d, ASSIGN}] 2024-11-13T17:59:21,443 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=dc926cd9c7c1f78f2949ef7a52bf7b2d, ASSIGN 2024-11-13T17:59:21,445 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=dc926cd9c7c1f78f2949ef7a52bf7b2d, ASSIGN; state=OFFLINE, location=39e84130bbc9,43723,1731520689798; forceNewPlan=false, retain=false 2024-11-13T17:59:21,449 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-11-13T17:59:21,596 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=dc926cd9c7c1f78f2949ef7a52bf7b2d, regionState=OPENING, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:21,600 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=dc926cd9c7c1f78f2949ef7a52bf7b2d, ASSIGN because future has completed 2024-11-13T17:59:21,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc926cd9c7c1f78f2949ef7a52bf7b2d, server=39e84130bbc9,43723,1731520689798}] 2024-11-13T17:59:21,761 INFO [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d. 2024-11-13T17:59:21,761 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => dc926cd9c7c1f78f2949ef7a52bf7b2d, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d.', STARTKEY => '', ENDKEY => ''} 2024-11-13T17:59:21,761 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync dc926cd9c7c1f78f2949ef7a52bf7b2d 2024-11-13T17:59:21,762 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:59:21,762 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for dc926cd9c7c1f78f2949ef7a52bf7b2d 2024-11-13T17:59:21,762 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for dc926cd9c7c1f78f2949ef7a52bf7b2d 2024-11-13T17:59:21,764 INFO [StoreOpener-dc926cd9c7c1f78f2949ef7a52bf7b2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region dc926cd9c7c1f78f2949ef7a52bf7b2d 2024-11-13T17:59:21,766 INFO [StoreOpener-dc926cd9c7c1f78f2949ef7a52bf7b2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc926cd9c7c1f78f2949ef7a52bf7b2d columnFamilyName testFamily 2024-11-13T17:59:21,766 DEBUG [StoreOpener-dc926cd9c7c1f78f2949ef7a52bf7b2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:21,767 INFO [StoreOpener-dc926cd9c7c1f78f2949ef7a52bf7b2d-1 {}] regionserver.HStore(327): Store=dc926cd9c7c1f78f2949ef7a52bf7b2d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T17:59:21,767 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for dc926cd9c7c1f78f2949ef7a52bf7b2d 2024-11-13T17:59:21,768 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnNextAsync/dc926cd9c7c1f78f2949ef7a52bf7b2d 2024-11-13T17:59:21,768 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnNextAsync/dc926cd9c7c1f78f2949ef7a52bf7b2d 2024-11-13T17:59:21,769 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for dc926cd9c7c1f78f2949ef7a52bf7b2d 2024-11-13T17:59:21,769 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for dc926cd9c7c1f78f2949ef7a52bf7b2d 2024-11-13T17:59:21,772 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for dc926cd9c7c1f78f2949ef7a52bf7b2d 2024-11-13T17:59:21,775 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnNextAsync/dc926cd9c7c1f78f2949ef7a52bf7b2d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T17:59:21,776 INFO [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened dc926cd9c7c1f78f2949ef7a52bf7b2d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70933589, jitterRate=0.05699284374713898}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T17:59:21,776 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dc926cd9c7c1f78f2949ef7a52bf7b2d 2024-11-13T17:59:21,777 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for dc926cd9c7c1f78f2949ef7a52bf7b2d: Running coprocessor pre-open hook at 1731520761762Writing region info on filesystem at 1731520761762Initializing all the Stores at 1731520761763 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731520761763Cleaning up temporary data from old regions at 1731520761769 (+6 ms)Running coprocessor post-open hooks at 1731520761776 (+7 ms)Region opened successfully at 1731520761777 (+1 ms) 2024-11-13T17:59:21,778 INFO [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d., pid=12, masterSystemTime=1731520761755 2024-11-13T17:59:21,781 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d. 2024-11-13T17:59:21,781 INFO [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d. 2024-11-13T17:59:21,782 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=dc926cd9c7c1f78f2949ef7a52bf7b2d, regionState=OPEN, openSeqNum=2, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:21,785 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc926cd9c7c1f78f2949ef7a52bf7b2d, server=39e84130bbc9,43723,1731520689798 because future has completed 2024-11-13T17:59:21,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-13T17:59:21,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure dc926cd9c7c1f78f2949ef7a52bf7b2d, server=39e84130bbc9,43723,1731520689798 in 186 msec 2024-11-13T17:59:21,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-13T17:59:21,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=dc926cd9c7c1f78f2949ef7a52bf7b2d, ASSIGN in 351 msec 2024-11-13T17:59:21,795 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T17:59:21,795 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731520761795"}]},"ts":"1731520761795"} 2024-11-13T17:59:21,798 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-13T17:59:21,799 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T17:59:21,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 809 msec 2024-11-13T17:59:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-13T17:59:22,002 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-13T17:59:22,002 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-13T17:59:22,002 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T17:59:22,007 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-13T17:59:22,007 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T17:59:22,008 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-13T17:59:22,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d., hostname=39e84130bbc9,43723,1731520689798, seqNum=2] 2024-11-13T17:59:22,025 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-13T17:59:22,026 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-13T17:59:22,029 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-13T17:59:22,031 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-13T17:59:22,033 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-13T17:59:22,033 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43723 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-13T17:59:23,743 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-13T17:59:27,264 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T17:59:28,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43723 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '3409586232043241493' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-13T17:59:28,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43723 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:47138 deadline: 1731520774037, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '3409586232043241493' does not exist 2024-11-13T17:59:28,042 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 39e84130bbc9,43723,1731520689798 for closing scanner id = 3409586232043241493 for dc926cd9c7c1f78f2949ef7a52bf7b2d of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '3409586232043241493' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T17:59:28,046 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=220 (was 218) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=468 (was 508), ProcessCount=11 (was 11), AvailableMemoryMB=3273 (was 2404) - AvailableMemoryMB LEAK? - 2024-11-13T17:59:28,057 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=220, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=468, ProcessCount=11, AvailableMemoryMB=3270 2024-11-13T17:59:28,061 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T17:59:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-13T17:59:28,065 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T17:59:28,065 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:28,066 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-13T17:59:28,067 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T17:59:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T17:59:28,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741841_1017 (size=423) 2024-11-13T17:59:28,488 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 279f8f9c9faa5090c68e4d3f7af73bb8, NAME => 'testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0 2024-11-13T17:59:28,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741842_1018 (size=74) 2024-11-13T17:59:28,516 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:59:28,516 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 279f8f9c9faa5090c68e4d3f7af73bb8, disabling compactions & flushes 2024-11-13T17:59:28,516 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8. 2024-11-13T17:59:28,516 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8. 2024-11-13T17:59:28,516 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8. after waiting 0 ms 2024-11-13T17:59:28,516 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8. 2024-11-13T17:59:28,516 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8. 2024-11-13T17:59:28,516 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 279f8f9c9faa5090c68e4d3f7af73bb8: Waiting for close lock at 1731520768516Disabling compacts and flushes for region at 1731520768516Disabling writes for close at 1731520768516Writing region close event to WAL at 1731520768516Closed at 1731520768516 2024-11-13T17:59:28,520 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T17:59:28,520 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731520768520"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731520768520"}]},"ts":"1731520768520"} 2024-11-13T17:59:28,525 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T17:59:28,527 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T17:59:28,527 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731520768527"}]},"ts":"1731520768527"} 2024-11-13T17:59:28,532 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-13T17:59:28,533 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=279f8f9c9faa5090c68e4d3f7af73bb8, ASSIGN}] 2024-11-13T17:59:28,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-11-13T17:59:28,535 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=279f8f9c9faa5090c68e4d3f7af73bb8, ASSIGN 2024-11-13T17:59:28,537 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=279f8f9c9faa5090c68e4d3f7af73bb8, ASSIGN; state=OFFLINE, location=39e84130bbc9,43723,1731520689798; forceNewPlan=false, retain=false 2024-11-13T17:59:28,688 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=279f8f9c9faa5090c68e4d3f7af73bb8, regionState=OPENING, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:28,693 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=279f8f9c9faa5090c68e4d3f7af73bb8, ASSIGN because future has completed 2024-11-13T17:59:28,694 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 279f8f9c9faa5090c68e4d3f7af73bb8, server=39e84130bbc9,43723,1731520689798}] 2024-11-13T17:59:28,855 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8. 2024-11-13T17:59:28,855 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 279f8f9c9faa5090c68e4d3f7af73bb8, NAME => 'testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8.', STARTKEY => '', ENDKEY => ''} 2024-11-13T17:59:28,855 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 279f8f9c9faa5090c68e4d3f7af73bb8 2024-11-13T17:59:28,856 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:59:28,856 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 279f8f9c9faa5090c68e4d3f7af73bb8 2024-11-13T17:59:28,856 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 279f8f9c9faa5090c68e4d3f7af73bb8 2024-11-13T17:59:28,858 INFO [StoreOpener-279f8f9c9faa5090c68e4d3f7af73bb8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 279f8f9c9faa5090c68e4d3f7af73bb8 2024-11-13T17:59:28,862 INFO [StoreOpener-279f8f9c9faa5090c68e4d3f7af73bb8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 279f8f9c9faa5090c68e4d3f7af73bb8 columnFamilyName testFamily 2024-11-13T17:59:28,862 DEBUG [StoreOpener-279f8f9c9faa5090c68e4d3f7af73bb8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:28,863 INFO [StoreOpener-279f8f9c9faa5090c68e4d3f7af73bb8-1 {}] regionserver.HStore(327): Store=279f8f9c9faa5090c68e4d3f7af73bb8/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T17:59:28,863 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 279f8f9c9faa5090c68e4d3f7af73bb8 2024-11-13T17:59:28,864 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testRetryOutOfOrderScannerNextException/279f8f9c9faa5090c68e4d3f7af73bb8 2024-11-13T17:59:28,866 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testRetryOutOfOrderScannerNextException/279f8f9c9faa5090c68e4d3f7af73bb8 2024-11-13T17:59:28,866 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 279f8f9c9faa5090c68e4d3f7af73bb8 2024-11-13T17:59:28,866 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 279f8f9c9faa5090c68e4d3f7af73bb8 2024-11-13T17:59:28,870 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 279f8f9c9faa5090c68e4d3f7af73bb8 2024-11-13T17:59:28,878 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testRetryOutOfOrderScannerNextException/279f8f9c9faa5090c68e4d3f7af73bb8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T17:59:28,879 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 279f8f9c9faa5090c68e4d3f7af73bb8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69569000, jitterRate=0.0366588830947876}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T17:59:28,880 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 279f8f9c9faa5090c68e4d3f7af73bb8 2024-11-13T17:59:28,881 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 279f8f9c9faa5090c68e4d3f7af73bb8: Running coprocessor pre-open hook at 1731520768856Writing region info on filesystem at 1731520768856Initializing all the Stores at 1731520768858 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731520768858Cleaning up temporary data from old regions at 1731520768866 (+8 ms)Running coprocessor post-open hooks at 1731520768880 (+14 ms)Region opened successfully at 1731520768881 (+1 ms) 2024-11-13T17:59:28,883 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8., pid=15, masterSystemTime=1731520768848 2024-11-13T17:59:28,889 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8. 2024-11-13T17:59:28,889 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8. 2024-11-13T17:59:28,890 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=279f8f9c9faa5090c68e4d3f7af73bb8, regionState=OPEN, openSeqNum=2, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:28,893 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 279f8f9c9faa5090c68e4d3f7af73bb8, server=39e84130bbc9,43723,1731520689798 because future has completed 2024-11-13T17:59:28,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-13T17:59:28,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 279f8f9c9faa5090c68e4d3f7af73bb8, server=39e84130bbc9,43723,1731520689798 in 202 msec 2024-11-13T17:59:28,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T17:59:28,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=279f8f9c9faa5090c68e4d3f7af73bb8, ASSIGN in 372 msec 2024-11-13T17:59:28,911 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T17:59:28,911 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731520768911"}]},"ts":"1731520768911"} 2024-11-13T17:59:28,922 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-13T17:59:28,924 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T17:59:28,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 863 msec 2024-11-13T17:59:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T17:59:29,082 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-13T17:59:29,082 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-13T17:59:29,082 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T17:59:29,088 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-13T17:59:29,088 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T17:59:29,088 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-13T17:59:29,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8., hostname=39e84130bbc9,43723,1731520689798, seqNum=2] 2024-11-13T17:59:29,106 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-13T17:59:29,106 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-13T17:59:29,111 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-13T17:59:29,113 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-13T17:59:29,115 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-13T17:59:29,116 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=3409586232043241497 2024-11-13T17:59:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:47138 deadline: 1731520772114, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-13T17:59:29,120 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-13T17:59:29,121 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-13T17:59:29,122 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-13T17:59:29,123 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-13T17:59:29,125 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=3409586232043241499 2024-11-13T17:59:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:47138 deadline: 1731520772123, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-13T17:59:29,129 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-13T17:59:29,130 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=3409586232043241500 2024-11-13T17:59:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:47138 deadline: 1731520772128, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-13T17:59:29,134 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-13T17:59:29,135 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=3409586232043241501 2024-11-13T17:59:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:47138 deadline: 1731520772134, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-13T17:59:29,139 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-13T17:59:29,152 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 220) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1042495474_22 at /127.0.0.1:46214 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=468 (was 468), ProcessCount=11 (was 11), AvailableMemoryMB=3054 (was 3270) 2024-11-13T17:59:29,163 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=468, ProcessCount=11, AvailableMemoryMB=3053 2024-11-13T17:59:29,167 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T17:59:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-13T17:59:29,170 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T17:59:29,171 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:29,171 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-13T17:59:29,172 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T17:59:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-13T17:59:29,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741843_1019 (size=418) 2024-11-13T17:59:29,184 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d83adb705607529a1edc955b54ca2574, NAME => 'testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0 2024-11-13T17:59:29,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741844_1020 (size=69) 2024-11-13T17:59:29,193 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:59:29,193 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing d83adb705607529a1edc955b54ca2574, disabling compactions & flushes 2024-11-13T17:59:29,193 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574. 2024-11-13T17:59:29,193 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574. 2024-11-13T17:59:29,193 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574. after waiting 0 ms 2024-11-13T17:59:29,193 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574. 2024-11-13T17:59:29,193 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574. 2024-11-13T17:59:29,193 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for d83adb705607529a1edc955b54ca2574: Waiting for close lock at 1731520769193Disabling compacts and flushes for region at 1731520769193Disabling writes for close at 1731520769193Writing region close event to WAL at 1731520769193Closed at 1731520769193 2024-11-13T17:59:29,195 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T17:59:29,195 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1731520769195"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731520769195"}]},"ts":"1731520769195"} 2024-11-13T17:59:29,198 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T17:59:29,200 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T17:59:29,200 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731520769200"}]},"ts":"1731520769200"} 2024-11-13T17:59:29,203 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-13T17:59:29,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d83adb705607529a1edc955b54ca2574, ASSIGN}] 2024-11-13T17:59:29,205 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d83adb705607529a1edc955b54ca2574, ASSIGN 2024-11-13T17:59:29,206 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d83adb705607529a1edc955b54ca2574, ASSIGN; state=OFFLINE, location=39e84130bbc9,43723,1731520689798; forceNewPlan=false, retain=false 2024-11-13T17:59:29,259 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-13T17:59:29,259 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-13T17:59:29,261 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-13T17:59:29,261 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-13T17:59:29,357 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=d83adb705607529a1edc955b54ca2574, regionState=OPENING, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:29,361 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d83adb705607529a1edc955b54ca2574, ASSIGN because future has completed 2024-11-13T17:59:29,361 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure d83adb705607529a1edc955b54ca2574, server=39e84130bbc9,43723,1731520689798}] 2024-11-13T17:59:29,521 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574. 2024-11-13T17:59:29,521 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => d83adb705607529a1edc955b54ca2574, NAME => 'testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574.', STARTKEY => '', ENDKEY => ''} 2024-11-13T17:59:29,521 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner d83adb705607529a1edc955b54ca2574 2024-11-13T17:59:29,522 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:59:29,522 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for d83adb705607529a1edc955b54ca2574 2024-11-13T17:59:29,522 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for d83adb705607529a1edc955b54ca2574 2024-11-13T17:59:29,524 INFO [StoreOpener-d83adb705607529a1edc955b54ca2574-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d83adb705607529a1edc955b54ca2574 2024-11-13T17:59:29,526 INFO [StoreOpener-d83adb705607529a1edc955b54ca2574-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d83adb705607529a1edc955b54ca2574 columnFamilyName testFamily 2024-11-13T17:59:29,526 DEBUG [StoreOpener-d83adb705607529a1edc955b54ca2574-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:29,527 INFO [StoreOpener-d83adb705607529a1edc955b54ca2574-1 {}] regionserver.HStore(327): Store=d83adb705607529a1edc955b54ca2574/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T17:59:29,527 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for d83adb705607529a1edc955b54ca2574 2024-11-13T17:59:29,528 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnOpenScanner/d83adb705607529a1edc955b54ca2574 2024-11-13T17:59:29,528 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnOpenScanner/d83adb705607529a1edc955b54ca2574 2024-11-13T17:59:29,529 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for d83adb705607529a1edc955b54ca2574 2024-11-13T17:59:29,529 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for d83adb705607529a1edc955b54ca2574 2024-11-13T17:59:29,531 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for d83adb705607529a1edc955b54ca2574 2024-11-13T17:59:29,534 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnOpenScanner/d83adb705607529a1edc955b54ca2574/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T17:59:29,535 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened d83adb705607529a1edc955b54ca2574; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70855668, jitterRate=0.055831730365753174}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T17:59:29,535 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d83adb705607529a1edc955b54ca2574 2024-11-13T17:59:29,536 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for d83adb705607529a1edc955b54ca2574: Running coprocessor pre-open hook at 1731520769522Writing region info on filesystem at 1731520769522Initializing all the Stores at 1731520769524 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731520769524Cleaning up temporary data from old regions at 1731520769529 (+5 ms)Running coprocessor post-open hooks at 1731520769535 (+6 ms)Region opened successfully at 1731520769536 (+1 ms) 2024-11-13T17:59:29,537 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574., pid=18, masterSystemTime=1731520769514 2024-11-13T17:59:29,540 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574. 2024-11-13T17:59:29,540 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574. 2024-11-13T17:59:29,541 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=d83adb705607529a1edc955b54ca2574, regionState=OPEN, openSeqNum=2, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:29,545 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure d83adb705607529a1edc955b54ca2574, server=39e84130bbc9,43723,1731520689798 because future has completed 2024-11-13T17:59:29,548 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-13T17:59:29,549 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure d83adb705607529a1edc955b54ca2574, server=39e84130bbc9,43723,1731520689798 in 185 msec 2024-11-13T17:59:29,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-13T17:59:29,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d83adb705607529a1edc955b54ca2574, ASSIGN in 345 msec 2024-11-13T17:59:29,553 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T17:59:29,553 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731520769553"}]},"ts":"1731520769553"} 2024-11-13T17:59:29,556 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-13T17:59:29,557 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T17:59:29,560 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 390 msec 2024-11-13T17:59:29,745 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-13T17:59:29,745 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-13T17:59:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-13T17:59:30,181 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-13T17:59:30,181 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-13T17:59:30,182 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T17:59:30,188 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-13T17:59:30,188 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T17:59:30,188 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-13T17:59:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574., hostname=39e84130bbc9,43723,1731520689798, seqNum=2] 2024-11-13T17:59:30,210 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-13T17:59:30,210 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-13T17:59:30,215 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-13T17:59:30,217 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-13T17:59:31,222 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574., hostname=39e84130bbc9,43723,1731520689798, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574., hostname=39e84130bbc9,43723,1731520689798, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=39e84130bbc9:43723 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-13T17:59:31,223 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574., hostname=39e84130bbc9,43723,1731520689798, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-13T17:59:31,223 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574., hostname=39e84130bbc9,43723,1731520689798, seqNum=2 from cache 2024-11-13T17:59:31,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-11-13T17:59:32,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574., hostname=39e84130bbc9,43723,1731520689798, seqNum=2] 2024-11-13T17:59:32,235 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-13T17:59:33,222 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574., hostname=39e84130bbc9,43723,1731520689798, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574., hostname=39e84130bbc9,43723,1731520689798, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=39e84130bbc9:43723 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=988ms, rpcTimeout=981ms 2024-11-13T17:59:33,222 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574., hostname=39e84130bbc9,43723,1731520689798, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=988ms, rpcTimeout=981ms 2024-11-13T17:59:33,223 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574., hostname=39e84130bbc9,43723,1731520689798, seqNum=2 from cache 2024-11-13T17:59:33,235 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223 (was 223), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=495 (was 468) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2803 (was 3053) 2024-11-13T17:59:33,245 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=495, ProcessCount=11, AvailableMemoryMB=2802 2024-11-13T17:59:33,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T17:59:33,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-13T17:59:33,253 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T17:59:33,253 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:33,253 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-13T17:59:33,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T17:59:33,255 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T17:59:33,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741845_1021 (size=416) 2024-11-13T17:59:33,268 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1b7c345630c8c5e80a6529162c393062, NAME => 'testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0 2024-11-13T17:59:33,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741846_1022 (size=67) 2024-11-13T17:59:33,280 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:59:33,280 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 1b7c345630c8c5e80a6529162c393062, disabling compactions & flushes 2024-11-13T17:59:33,281 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062. 2024-11-13T17:59:33,281 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062. 2024-11-13T17:59:33,281 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062. after waiting 0 ms 2024-11-13T17:59:33,281 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062. 2024-11-13T17:59:33,281 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062. 2024-11-13T17:59:33,281 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1b7c345630c8c5e80a6529162c393062: Waiting for close lock at 1731520773280Disabling compacts and flushes for region at 1731520773280Disabling writes for close at 1731520773281 (+1 ms)Writing region close event to WAL at 1731520773281Closed at 1731520773281 2024-11-13T17:59:33,283 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T17:59:33,284 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731520773283"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731520773283"}]},"ts":"1731520773283"} 2024-11-13T17:59:33,286 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T17:59:33,288 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T17:59:33,288 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731520773288"}]},"ts":"1731520773288"} 2024-11-13T17:59:33,290 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-13T17:59:33,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=1b7c345630c8c5e80a6529162c393062, ASSIGN}] 2024-11-13T17:59:33,292 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=1b7c345630c8c5e80a6529162c393062, ASSIGN 2024-11-13T17:59:33,293 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=1b7c345630c8c5e80a6529162c393062, ASSIGN; state=OFFLINE, location=39e84130bbc9,43723,1731520689798; forceNewPlan=false, retain=false 2024-11-13T17:59:33,444 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=1b7c345630c8c5e80a6529162c393062, regionState=OPENING, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:33,448 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=1b7c345630c8c5e80a6529162c393062, ASSIGN because future has completed 2024-11-13T17:59:33,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b7c345630c8c5e80a6529162c393062, server=39e84130bbc9,43723,1731520689798}] 2024-11-13T17:59:33,607 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062. 2024-11-13T17:59:33,607 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b7c345630c8c5e80a6529162c393062, NAME => 'testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062.', STARTKEY => '', ENDKEY => ''} 2024-11-13T17:59:33,608 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 1b7c345630c8c5e80a6529162c393062 2024-11-13T17:59:33,608 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:59:33,608 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 1b7c345630c8c5e80a6529162c393062 2024-11-13T17:59:33,608 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 1b7c345630c8c5e80a6529162c393062 2024-11-13T17:59:33,610 INFO [StoreOpener-1b7c345630c8c5e80a6529162c393062-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1b7c345630c8c5e80a6529162c393062 2024-11-13T17:59:33,612 INFO [StoreOpener-1b7c345630c8c5e80a6529162c393062-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b7c345630c8c5e80a6529162c393062 columnFamilyName testFamily 2024-11-13T17:59:33,612 DEBUG [StoreOpener-1b7c345630c8c5e80a6529162c393062-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:33,612 INFO [StoreOpener-1b7c345630c8c5e80a6529162c393062-1 {}] regionserver.HStore(327): Store=1b7c345630c8c5e80a6529162c393062/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T17:59:33,613 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 1b7c345630c8c5e80a6529162c393062 2024-11-13T17:59:33,614 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnNextAsync/1b7c345630c8c5e80a6529162c393062 2024-11-13T17:59:33,614 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnNextAsync/1b7c345630c8c5e80a6529162c393062 2024-11-13T17:59:33,615 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 1b7c345630c8c5e80a6529162c393062 2024-11-13T17:59:33,615 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 1b7c345630c8c5e80a6529162c393062 2024-11-13T17:59:33,617 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 1b7c345630c8c5e80a6529162c393062 2024-11-13T17:59:33,620 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnNextAsync/1b7c345630c8c5e80a6529162c393062/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T17:59:33,620 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 1b7c345630c8c5e80a6529162c393062; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65976461, jitterRate=-0.016874119639396667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T17:59:33,621 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1b7c345630c8c5e80a6529162c393062 2024-11-13T17:59:33,621 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 1b7c345630c8c5e80a6529162c393062: Running coprocessor pre-open hook at 1731520773608Writing region info on filesystem at 1731520773608Initializing all the Stores at 1731520773609 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731520773609Cleaning up temporary data from old regions at 1731520773615 (+6 ms)Running coprocessor post-open hooks at 1731520773621 (+6 ms)Region opened successfully at 1731520773621 2024-11-13T17:59:33,622 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062., pid=21, masterSystemTime=1731520773601 2024-11-13T17:59:33,625 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062. 2024-11-13T17:59:33,625 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062. 2024-11-13T17:59:33,626 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=1b7c345630c8c5e80a6529162c393062, regionState=OPEN, openSeqNum=2, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:33,628 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b7c345630c8c5e80a6529162c393062, server=39e84130bbc9,43723,1731520689798 because future has completed 2024-11-13T17:59:33,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-13T17:59:33,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 1b7c345630c8c5e80a6529162c393062, server=39e84130bbc9,43723,1731520689798 in 181 msec 2024-11-13T17:59:33,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-13T17:59:33,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=1b7c345630c8c5e80a6529162c393062, ASSIGN in 341 msec 2024-11-13T17:59:33,635 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T17:59:33,636 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731520773636"}]},"ts":"1731520773636"} 2024-11-13T17:59:33,638 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-13T17:59:33,639 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T17:59:33,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 390 msec 2024-11-13T17:59:33,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-11-13T17:59:34,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T17:59:34,261 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-13T17:59:34,261 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-13T17:59:34,261 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T17:59:34,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-13T17:59:34,267 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T17:59:34,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-13T17:59:34,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062., hostname=39e84130bbc9,43723,1731520689798, seqNum=2] 2024-11-13T17:59:34,283 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-13T17:59:34,283 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-13T17:59:34,287 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-13T17:59:34,289 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-13T17:59:34,290 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-13T17:59:34,291 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-13T17:59:34,358 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T17:59:35,747 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-13T17:59:37,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43723 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '3409586232043241512' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-13T17:59:37,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43723 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:47138 deadline: 1731520778293, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '3409586232043241512' does not exist 2024-11-13T17:59:37,301 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 39e84130bbc9,43723,1731520689798 for closing scanner id = 3409586232043241512 for 1b7c345630c8c5e80a6529162c393062 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '3409586232043241512' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T17:59:37,306 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 223), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=495 (was 495), ProcessCount=11 (was 11), AvailableMemoryMB=2598 (was 2802) 2024-11-13T17:59:37,317 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=495, ProcessCount=11, AvailableMemoryMB=2597 2024-11-13T17:59:37,320 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T17:59:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-13T17:59:37,325 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T17:59:37,325 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:37,325 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-13T17:59:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-13T17:59:37,327 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T17:59:37,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741847_1023 (size=411) 2024-11-13T17:59:37,383 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1d7a32f0d7ece74c8bc407457fff622b, NAME => 'testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0 2024-11-13T17:59:37,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741848_1024 (size=62) 2024-11-13T17:59:37,672 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T17:59:37,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-11-13T17:59:37,834 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:59:37,835 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 1d7a32f0d7ece74c8bc407457fff622b, disabling compactions & flushes 2024-11-13T17:59:37,835 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b. 2024-11-13T17:59:37,835 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b. 2024-11-13T17:59:37,835 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b. after waiting 0 ms 2024-11-13T17:59:37,835 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b. 2024-11-13T17:59:37,835 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b. 2024-11-13T17:59:37,835 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1d7a32f0d7ece74c8bc407457fff622b: Waiting for close lock at 1731520777835Disabling compacts and flushes for region at 1731520777835Disabling writes for close at 1731520777835Writing region close event to WAL at 1731520777835Closed at 1731520777835 2024-11-13T17:59:37,837 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T17:59:37,838 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731520777837"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731520777837"}]},"ts":"1731520777837"} 2024-11-13T17:59:37,842 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T17:59:37,844 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T17:59:37,844 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731520777844"}]},"ts":"1731520777844"} 2024-11-13T17:59:37,847 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-13T17:59:37,848 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=1d7a32f0d7ece74c8bc407457fff622b, ASSIGN}] 2024-11-13T17:59:37,850 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=1d7a32f0d7ece74c8bc407457fff622b, ASSIGN 2024-11-13T17:59:37,851 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=1d7a32f0d7ece74c8bc407457fff622b, ASSIGN; state=OFFLINE, location=39e84130bbc9,43723,1731520689798; forceNewPlan=false, retain=false 2024-11-13T17:59:38,002 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=1d7a32f0d7ece74c8bc407457fff622b, regionState=OPENING, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:38,005 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=1d7a32f0d7ece74c8bc407457fff622b, ASSIGN because future has completed 2024-11-13T17:59:38,006 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d7a32f0d7ece74c8bc407457fff622b, server=39e84130bbc9,43723,1731520689798}] 2024-11-13T17:59:38,164 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b. 2024-11-13T17:59:38,164 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d7a32f0d7ece74c8bc407457fff622b, NAME => 'testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b.', STARTKEY => '', ENDKEY => ''} 2024-11-13T17:59:38,165 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 1d7a32f0d7ece74c8bc407457fff622b 2024-11-13T17:59:38,165 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:59:38,165 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 1d7a32f0d7ece74c8bc407457fff622b 2024-11-13T17:59:38,165 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 1d7a32f0d7ece74c8bc407457fff622b 2024-11-13T17:59:38,167 INFO [StoreOpener-1d7a32f0d7ece74c8bc407457fff622b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1d7a32f0d7ece74c8bc407457fff622b 2024-11-13T17:59:38,169 INFO [StoreOpener-1d7a32f0d7ece74c8bc407457fff622b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d7a32f0d7ece74c8bc407457fff622b columnFamilyName testFamily 2024-11-13T17:59:38,169 DEBUG [StoreOpener-1d7a32f0d7ece74c8bc407457fff622b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:38,170 INFO [StoreOpener-1d7a32f0d7ece74c8bc407457fff622b-1 {}] regionserver.HStore(327): Store=1d7a32f0d7ece74c8bc407457fff622b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T17:59:38,170 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 1d7a32f0d7ece74c8bc407457fff622b 2024-11-13T17:59:38,171 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnNext/1d7a32f0d7ece74c8bc407457fff622b 2024-11-13T17:59:38,172 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnNext/1d7a32f0d7ece74c8bc407457fff622b 2024-11-13T17:59:38,173 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 1d7a32f0d7ece74c8bc407457fff622b 2024-11-13T17:59:38,173 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 1d7a32f0d7ece74c8bc407457fff622b 2024-11-13T17:59:38,175 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 1d7a32f0d7ece74c8bc407457fff622b 2024-11-13T17:59:38,178 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnNext/1d7a32f0d7ece74c8bc407457fff622b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T17:59:38,179 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 1d7a32f0d7ece74c8bc407457fff622b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75203987, jitterRate=0.12062673270702362}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T17:59:38,179 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1d7a32f0d7ece74c8bc407457fff622b 2024-11-13T17:59:38,180 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 1d7a32f0d7ece74c8bc407457fff622b: Running coprocessor pre-open hook at 1731520778165Writing region info on filesystem at 1731520778165Initializing all the Stores at 1731520778167 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731520778167Cleaning up temporary data from old regions at 1731520778173 (+6 ms)Running coprocessor post-open hooks at 1731520778179 (+6 ms)Region opened successfully at 1731520778180 (+1 ms) 2024-11-13T17:59:38,181 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b., pid=24, masterSystemTime=1731520778158 2024-11-13T17:59:38,184 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b. 2024-11-13T17:59:38,184 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b. 2024-11-13T17:59:38,185 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=1d7a32f0d7ece74c8bc407457fff622b, regionState=OPEN, openSeqNum=2, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:38,187 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d7a32f0d7ece74c8bc407457fff622b, server=39e84130bbc9,43723,1731520689798 because future has completed 2024-11-13T17:59:38,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-13T17:59:38,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 1d7a32f0d7ece74c8bc407457fff622b, server=39e84130bbc9,43723,1731520689798 in 182 msec 2024-11-13T17:59:38,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-13T17:59:38,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=1d7a32f0d7ece74c8bc407457fff622b, ASSIGN in 343 msec 2024-11-13T17:59:38,196 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T17:59:38,196 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731520778196"}]},"ts":"1731520778196"} 2024-11-13T17:59:38,198 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-13T17:59:38,199 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T17:59:38,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 880 msec 2024-11-13T17:59:38,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-13T17:59:38,342 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-13T17:59:38,342 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-13T17:59:38,342 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T17:59:38,348 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-13T17:59:38,348 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T17:59:38,349 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-13T17:59:38,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b., hostname=39e84130bbc9,43723,1731520689798, seqNum=2] 2024-11-13T17:59:38,366 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-13T17:59:38,367 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-13T17:59:38,371 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-13T17:59:38,373 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-13T17:59:38,375 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-13T17:59:38,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-13T17:59:39,259 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-13T17:59:39,259 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-13T17:59:39,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-13T17:59:39,260 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-13T17:59:39,261 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-13T17:59:39,261 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-13T17:59:41,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43723 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '3409586232043241516' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-13T17:59:41,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43723 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:47138 deadline: 1731520782383, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '3409586232043241516' does not exist 2024-11-13T17:59:41,386 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 39e84130bbc9,43723,1731520689798 for closing scanner id = 3409586232043241516 for 1d7a32f0d7ece74c8bc407457fff622b of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '3409586232043241516' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T17:59:41,393 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223 (was 222) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1042495474_22 at /127.0.0.1:35882 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=495 (was 495), ProcessCount=11 (was 11), AvailableMemoryMB=2159 (was 2597) 2024-11-13T17:59:41,402 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=495, ProcessCount=11, AvailableMemoryMB=2159 2024-11-13T17:59:41,404 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35773 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T17:59:41,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35773 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-13T17:59:41,407 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T17:59:41,408 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:41,408 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35773 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-13T17:59:41,409 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T17:59:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T17:59:41,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741849_1025 (size=419) 2024-11-13T17:59:41,748 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-13T17:59:41,821 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 43936dcb94433de7b46dc65301fffb7e, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0 2024-11-13T17:59:41,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741850_1026 (size=70) 2024-11-13T17:59:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-11-13T17:59:42,231 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:59:42,231 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 43936dcb94433de7b46dc65301fffb7e, disabling compactions & flushes 2024-11-13T17:59:42,231 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e. 2024-11-13T17:59:42,231 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e. 2024-11-13T17:59:42,231 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e. after waiting 0 ms 2024-11-13T17:59:42,231 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e. 2024-11-13T17:59:42,232 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e. 2024-11-13T17:59:42,232 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 43936dcb94433de7b46dc65301fffb7e: Waiting for close lock at 1731520782231Disabling compacts and flushes for region at 1731520782231Disabling writes for close at 1731520782231Writing region close event to WAL at 1731520782232 (+1 ms)Closed at 1731520782232 2024-11-13T17:59:42,233 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T17:59:42,234 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1731520782234"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731520782234"}]},"ts":"1731520782234"} 2024-11-13T17:59:42,237 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T17:59:42,238 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T17:59:42,238 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731520782238"}]},"ts":"1731520782238"} 2024-11-13T17:59:42,241 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-13T17:59:42,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=43936dcb94433de7b46dc65301fffb7e, ASSIGN}] 2024-11-13T17:59:42,242 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=43936dcb94433de7b46dc65301fffb7e, ASSIGN 2024-11-13T17:59:42,243 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=43936dcb94433de7b46dc65301fffb7e, ASSIGN; state=OFFLINE, location=39e84130bbc9,43723,1731520689798; forceNewPlan=false, retain=false 2024-11-13T17:59:42,394 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=43936dcb94433de7b46dc65301fffb7e, regionState=OPENING, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:42,398 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=43936dcb94433de7b46dc65301fffb7e, ASSIGN because future has completed 2024-11-13T17:59:42,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43936dcb94433de7b46dc65301fffb7e, server=39e84130bbc9,43723,1731520689798}] 2024-11-13T17:59:42,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T17:59:42,558 INFO [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e. 2024-11-13T17:59:42,558 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 43936dcb94433de7b46dc65301fffb7e, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e.', STARTKEY => '', ENDKEY => ''} 2024-11-13T17:59:42,558 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 43936dcb94433de7b46dc65301fffb7e 2024-11-13T17:59:42,558 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:59:42,559 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 43936dcb94433de7b46dc65301fffb7e 2024-11-13T17:59:42,559 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 43936dcb94433de7b46dc65301fffb7e 2024-11-13T17:59:42,561 INFO [StoreOpener-43936dcb94433de7b46dc65301fffb7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 43936dcb94433de7b46dc65301fffb7e 2024-11-13T17:59:42,563 INFO [StoreOpener-43936dcb94433de7b46dc65301fffb7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43936dcb94433de7b46dc65301fffb7e columnFamilyName testFamily 2024-11-13T17:59:42,563 DEBUG [StoreOpener-43936dcb94433de7b46dc65301fffb7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:42,563 INFO [StoreOpener-43936dcb94433de7b46dc65301fffb7e-1 {}] regionserver.HStore(327): Store=43936dcb94433de7b46dc65301fffb7e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T17:59:42,564 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 43936dcb94433de7b46dc65301fffb7e 2024-11-13T17:59:42,565 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/43936dcb94433de7b46dc65301fffb7e 2024-11-13T17:59:42,565 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/43936dcb94433de7b46dc65301fffb7e 2024-11-13T17:59:42,566 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 43936dcb94433de7b46dc65301fffb7e 2024-11-13T17:59:42,566 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 43936dcb94433de7b46dc65301fffb7e 2024-11-13T17:59:42,568 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 43936dcb94433de7b46dc65301fffb7e 2024-11-13T17:59:42,572 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/43936dcb94433de7b46dc65301fffb7e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T17:59:42,573 INFO [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 43936dcb94433de7b46dc65301fffb7e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70557659, jitterRate=0.05139105021953583}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T17:59:42,573 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 43936dcb94433de7b46dc65301fffb7e 2024-11-13T17:59:42,573 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 43936dcb94433de7b46dc65301fffb7e: Running coprocessor pre-open hook at 1731520782559Writing region info on filesystem at 1731520782559Initializing all the Stores at 1731520782560 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731520782560Cleaning up temporary data from old regions at 1731520782566 (+6 ms)Running coprocessor post-open hooks at 1731520782573 (+7 ms)Region opened successfully at 1731520782573 2024-11-13T17:59:42,575 INFO [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e., pid=27, masterSystemTime=1731520782552 2024-11-13T17:59:42,578 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e. 2024-11-13T17:59:42,578 INFO [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e. 2024-11-13T17:59:42,579 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=43936dcb94433de7b46dc65301fffb7e, regionState=OPEN, openSeqNum=2, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:42,581 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43936dcb94433de7b46dc65301fffb7e, server=39e84130bbc9,43723,1731520689798 because future has completed 2024-11-13T17:59:42,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-13T17:59:42,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 43936dcb94433de7b46dc65301fffb7e, server=39e84130bbc9,43723,1731520689798 in 183 msec 2024-11-13T17:59:42,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-13T17:59:42,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=43936dcb94433de7b46dc65301fffb7e, ASSIGN in 344 msec 2024-11-13T17:59:42,589 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T17:59:42,589 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731520782589"}]},"ts":"1731520782589"} 2024-11-13T17:59:42,592 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-13T17:59:42,593 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T17:59:42,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.1890 sec 2024-11-13T17:59:43,667 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T17:59:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T17:59:44,432 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-13T17:59:44,432 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-13T17:59:44,432 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T17:59:44,438 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-13T17:59:44,439 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T17:59:44,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-13T17:59:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e., hostname=39e84130bbc9,43723,1731520689798, seqNum=2] 2024-11-13T17:59:44,455 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-13T17:59:44,455 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-13T17:59:44,459 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-13T17:59:44,461 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43723 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-13T17:59:47,750 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-13T17:59:49,259 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-13T17:59:49,259 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-13T17:59:50,463 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e., hostname=39e84130bbc9,43723,1731520689798, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e., hostname=39e84130bbc9,43723,1731520689798, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=39e84130bbc9:43723 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6002ms, rpcTimeout=5999ms 2024-11-13T17:59:50,463 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e., hostname=39e84130bbc9,43723,1731520689798, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6002ms, rpcTimeout=5999ms 2024-11-13T17:59:50,463 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e., hostname=39e84130bbc9,43723,1731520689798, seqNum=2 from cache 2024-11-13T17:59:50,474 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 223), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=466 (was 495), ProcessCount=11 (was 11), AvailableMemoryMB=1770 (was 2159) 2024-11-13T17:59:50,484 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=466, ProcessCount=11, AvailableMemoryMB=1770 2024-11-13T17:59:50,487 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35773 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T17:59:50,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35773 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-13T17:59:50,491 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T17:59:50,491 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:50,491 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35773 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-13T17:59:50,492 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T17:59:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-13T17:59:50,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741851_1027 (size=407) 2024-11-13T17:59:50,903 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 56e1e10d2d6d43938ddcb116091e6585, NAME => 'hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0 2024-11-13T17:59:50,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741852_1028 (size=58) 2024-11-13T17:59:50,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-11-13T17:59:51,314 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:59:51,314 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 56e1e10d2d6d43938ddcb116091e6585, disabling compactions & flushes 2024-11-13T17:59:51,314 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585. 2024-11-13T17:59:51,314 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585. 2024-11-13T17:59:51,314 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585. after waiting 0 ms 2024-11-13T17:59:51,314 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585. 2024-11-13T17:59:51,314 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585. 2024-11-13T17:59:51,314 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 56e1e10d2d6d43938ddcb116091e6585: Waiting for close lock at 1731520791314Disabling compacts and flushes for region at 1731520791314Disabling writes for close at 1731520791314Writing region close event to WAL at 1731520791314Closed at 1731520791314 2024-11-13T17:59:51,316 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T17:59:51,316 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1731520791316"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731520791316"}]},"ts":"1731520791316"} 2024-11-13T17:59:51,319 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T17:59:51,320 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T17:59:51,320 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731520791320"}]},"ts":"1731520791320"} 2024-11-13T17:59:51,323 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-13T17:59:51,324 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=56e1e10d2d6d43938ddcb116091e6585, ASSIGN}] 2024-11-13T17:59:51,325 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=56e1e10d2d6d43938ddcb116091e6585, ASSIGN 2024-11-13T17:59:51,326 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=56e1e10d2d6d43938ddcb116091e6585, ASSIGN; state=OFFLINE, location=39e84130bbc9,43723,1731520689798; forceNewPlan=false, retain=false 2024-11-13T17:59:51,477 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=56e1e10d2d6d43938ddcb116091e6585, regionState=OPENING, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:51,480 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=56e1e10d2d6d43938ddcb116091e6585, ASSIGN because future has completed 2024-11-13T17:59:51,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56e1e10d2d6d43938ddcb116091e6585, server=39e84130bbc9,43723,1731520689798}] 2024-11-13T17:59:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-13T17:59:51,640 INFO [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585. 2024-11-13T17:59:51,641 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 56e1e10d2d6d43938ddcb116091e6585, NAME => 'hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585.', STARTKEY => '', ENDKEY => ''} 2024-11-13T17:59:51,642 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 56e1e10d2d6d43938ddcb116091e6585 2024-11-13T17:59:51,642 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T17:59:51,642 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 56e1e10d2d6d43938ddcb116091e6585 2024-11-13T17:59:51,642 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 56e1e10d2d6d43938ddcb116091e6585 2024-11-13T17:59:51,644 INFO [StoreOpener-56e1e10d2d6d43938ddcb116091e6585-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 56e1e10d2d6d43938ddcb116091e6585 2024-11-13T17:59:51,647 INFO [StoreOpener-56e1e10d2d6d43938ddcb116091e6585-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56e1e10d2d6d43938ddcb116091e6585 columnFamilyName testFamily 2024-11-13T17:59:51,647 DEBUG [StoreOpener-56e1e10d2d6d43938ddcb116091e6585-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:51,648 INFO [StoreOpener-56e1e10d2d6d43938ddcb116091e6585-1 {}] regionserver.HStore(327): Store=56e1e10d2d6d43938ddcb116091e6585/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T17:59:51,648 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 56e1e10d2d6d43938ddcb116091e6585 2024-11-13T17:59:51,649 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnNext/56e1e10d2d6d43938ddcb116091e6585 2024-11-13T17:59:51,650 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnNext/56e1e10d2d6d43938ddcb116091e6585 2024-11-13T17:59:51,651 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 56e1e10d2d6d43938ddcb116091e6585 2024-11-13T17:59:51,651 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 56e1e10d2d6d43938ddcb116091e6585 2024-11-13T17:59:51,653 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 56e1e10d2d6d43938ddcb116091e6585 2024-11-13T17:59:51,657 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnNext/56e1e10d2d6d43938ddcb116091e6585/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T17:59:51,658 INFO [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 56e1e10d2d6d43938ddcb116091e6585; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66977825, jitterRate=-0.001952633261680603}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T17:59:51,659 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 56e1e10d2d6d43938ddcb116091e6585 2024-11-13T17:59:51,659 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 56e1e10d2d6d43938ddcb116091e6585: Running coprocessor pre-open hook at 1731520791642Writing region info on filesystem at 1731520791642Initializing all the Stores at 1731520791644 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731520791644Cleaning up temporary data from old regions at 1731520791651 (+7 ms)Running coprocessor post-open hooks at 1731520791659 (+8 ms)Region opened successfully at 1731520791659 2024-11-13T17:59:51,661 INFO [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585., pid=30, masterSystemTime=1731520791633 2024-11-13T17:59:51,664 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585. 2024-11-13T17:59:51,664 INFO [RS_OPEN_PRIORITY_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585. 2024-11-13T17:59:51,665 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=56e1e10d2d6d43938ddcb116091e6585, regionState=OPEN, openSeqNum=2, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T17:59:51,668 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56e1e10d2d6d43938ddcb116091e6585, server=39e84130bbc9,43723,1731520689798 because future has completed 2024-11-13T17:59:51,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-13T17:59:51,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 56e1e10d2d6d43938ddcb116091e6585, server=39e84130bbc9,43723,1731520689798 in 189 msec 2024-11-13T17:59:51,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-13T17:59:51,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=56e1e10d2d6d43938ddcb116091e6585, ASSIGN in 349 msec 2024-11-13T17:59:51,677 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T17:59:51,678 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731520791677"}]},"ts":"1731520791677"} 2024-11-13T17:59:51,680 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-13T17:59:51,681 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T17:59:51,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.1950 sec 2024-11-13T17:59:53,141 INFO [master/39e84130bbc9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T17:59:53,141 INFO [master/39e84130bbc9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T17:59:53,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-13T17:59:53,532 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-13T17:59:53,532 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-13T17:59:53,532 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T17:59:53,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-13T17:59:53,539 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T17:59:53,539 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-13T17:59:53,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585., hostname=39e84130bbc9,43723,1731520689798, seqNum=2] 2024-11-13T17:59:53,555 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-13T17:59:53,556 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-13T17:59:53,559 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-13T17:59:53,562 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-13T17:59:53,563 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-13T17:59:53,564 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43723 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-13T17:59:53,751 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-13T17:59:57,176 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T17:59:59,259 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-13T17:59:59,259 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-13T17:59:59,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43723 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '3409586232043241524' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-13T17:59:59,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43723 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:47138 deadline: 1731520805574, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '3409586232043241524' does not exist 2024-11-13T17:59:59,577 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 39e84130bbc9,43723,1731520689798 for closing scanner id = 3409586232043241524 for 56e1e10d2d6d43938ddcb116091e6585 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '3409586232043241524' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T17:59:59,585 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=447 (was 466), ProcessCount=11 (was 11), AvailableMemoryMB=1438 (was 1770) 2024-11-13T17:59:59,594 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=447, ProcessCount=11, AvailableMemoryMB=1438 2024-11-13T17:59:59,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T17:59:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-13T17:59:59,601 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T17:59:59,601 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T17:59:59,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-13T17:59:59,602 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T17:59:59,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-13T17:59:59,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741853_1029 (size=423) 2024-11-13T18:00:00,015 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 56ff372ec0bdb2b139d60268e9e63346, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0 2024-11-13T18:00:00,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741854_1030 (size=74) 2024-11-13T18:00:00,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-11-13T18:00:00,426 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:00:00,427 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 56ff372ec0bdb2b139d60268e9e63346, disabling compactions & flushes 2024-11-13T18:00:00,427 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346. 2024-11-13T18:00:00,427 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346. 2024-11-13T18:00:00,427 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346. after waiting 0 ms 2024-11-13T18:00:00,427 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346. 2024-11-13T18:00:00,427 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346. 2024-11-13T18:00:00,427 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 56ff372ec0bdb2b139d60268e9e63346: Waiting for close lock at 1731520800427Disabling compacts and flushes for region at 1731520800427Disabling writes for close at 1731520800427Writing region close event to WAL at 1731520800427Closed at 1731520800427 2024-11-13T18:00:00,429 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:00:00,429 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731520800429"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731520800429"}]},"ts":"1731520800429"} 2024-11-13T18:00:00,432 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:00:00,433 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:00:00,434 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731520800434"}]},"ts":"1731520800434"} 2024-11-13T18:00:00,436 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-13T18:00:00,436 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=56ff372ec0bdb2b139d60268e9e63346, ASSIGN}] 2024-11-13T18:00:00,438 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=56ff372ec0bdb2b139d60268e9e63346, ASSIGN 2024-11-13T18:00:00,439 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=56ff372ec0bdb2b139d60268e9e63346, ASSIGN; state=OFFLINE, location=39e84130bbc9,43723,1731520689798; forceNewPlan=false, retain=false 2024-11-13T18:00:00,589 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=56ff372ec0bdb2b139d60268e9e63346, regionState=OPENING, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T18:00:00,593 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=56ff372ec0bdb2b139d60268e9e63346, ASSIGN because future has completed 2024-11-13T18:00:00,594 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56ff372ec0bdb2b139d60268e9e63346, server=39e84130bbc9,43723,1731520689798}] 2024-11-13T18:00:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-13T18:00:00,752 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346. 2024-11-13T18:00:00,753 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 56ff372ec0bdb2b139d60268e9e63346, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:00:00,753 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 56ff372ec0bdb2b139d60268e9e63346 2024-11-13T18:00:00,753 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:00:00,753 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 56ff372ec0bdb2b139d60268e9e63346 2024-11-13T18:00:00,753 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 56ff372ec0bdb2b139d60268e9e63346 2024-11-13T18:00:00,755 INFO [StoreOpener-56ff372ec0bdb2b139d60268e9e63346-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 56ff372ec0bdb2b139d60268e9e63346 2024-11-13T18:00:00,757 INFO [StoreOpener-56ff372ec0bdb2b139d60268e9e63346-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ff372ec0bdb2b139d60268e9e63346 columnFamilyName testFamily 2024-11-13T18:00:00,757 DEBUG [StoreOpener-56ff372ec0bdb2b139d60268e9e63346-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:00:00,758 INFO [StoreOpener-56ff372ec0bdb2b139d60268e9e63346-1 {}] regionserver.HStore(327): Store=56ff372ec0bdb2b139d60268e9e63346/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:00:00,758 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 56ff372ec0bdb2b139d60268e9e63346 2024-11-13T18:00:00,759 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnOpenScannerAsync/56ff372ec0bdb2b139d60268e9e63346 2024-11-13T18:00:00,760 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnOpenScannerAsync/56ff372ec0bdb2b139d60268e9e63346 2024-11-13T18:00:00,761 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 56ff372ec0bdb2b139d60268e9e63346 2024-11-13T18:00:00,761 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 56ff372ec0bdb2b139d60268e9e63346 2024-11-13T18:00:00,763 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 56ff372ec0bdb2b139d60268e9e63346 2024-11-13T18:00:00,766 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnOpenScannerAsync/56ff372ec0bdb2b139d60268e9e63346/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:00:00,767 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 56ff372ec0bdb2b139d60268e9e63346; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69508429, jitterRate=0.03575630486011505}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T18:00:00,767 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 56ff372ec0bdb2b139d60268e9e63346 2024-11-13T18:00:00,768 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 56ff372ec0bdb2b139d60268e9e63346: Running coprocessor pre-open hook at 1731520800753Writing region info on filesystem at 1731520800753Initializing all the Stores at 1731520800755 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731520800755Cleaning up temporary data from old regions at 1731520800761 (+6 ms)Running coprocessor post-open hooks at 1731520800767 (+6 ms)Region opened successfully at 1731520800768 (+1 ms) 2024-11-13T18:00:00,769 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346., pid=33, masterSystemTime=1731520800746 2024-11-13T18:00:00,772 DEBUG [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346. 2024-11-13T18:00:00,772 INFO [RS_OPEN_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346. 2024-11-13T18:00:00,773 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=56ff372ec0bdb2b139d60268e9e63346, regionState=OPEN, openSeqNum=2, regionLocation=39e84130bbc9,43723,1731520689798 2024-11-13T18:00:00,775 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56ff372ec0bdb2b139d60268e9e63346, server=39e84130bbc9,43723,1731520689798 because future has completed 2024-11-13T18:00:00,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-13T18:00:00,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 56ff372ec0bdb2b139d60268e9e63346, server=39e84130bbc9,43723,1731520689798 in 183 msec 2024-11-13T18:00:00,782 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-13T18:00:00,782 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=56ff372ec0bdb2b139d60268e9e63346, ASSIGN in 343 msec 2024-11-13T18:00:00,783 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:00:00,783 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731520800783"}]},"ts":"1731520800783"} 2024-11-13T18:00:00,785 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-13T18:00:00,786 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:00:00,788 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.1890 sec 2024-11-13T18:00:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35773 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-13T18:00:02,641 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-13T18:00:02,641 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-13T18:00:02,642 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T18:00:02,647 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-13T18:00:02,647 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T18:00:02,647 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-13T18:00:02,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346., hostname=39e84130bbc9,43723,1731520689798, seqNum=2] 2024-11-13T18:00:02,661 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-13T18:00:02,661 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-13T18:00:02,664 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-13T18:00:02,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-13T18:00:03,673 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346., hostname=39e84130bbc9,43723,1731520689798, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346., hostname=39e84130bbc9,43723,1731520689798, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=39e84130bbc9:43723 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-11-13T18:00:03,673 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346., hostname=39e84130bbc9,43723,1731520689798, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-11-13T18:00:03,673 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346., hostname=39e84130bbc9,43723,1731520689798, seqNum=2 from cache 2024-11-13T18:00:04,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-11-13T18:00:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346., hostname=39e84130bbc9,43723,1731520689798, seqNum=2] 2024-11-13T18:00:04,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43723 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-13T18:00:05,673 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346., hostname=39e84130bbc9,43723,1731520689798, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346., hostname=39e84130bbc9,43723,1731520689798, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=39e84130bbc9:43723 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=989ms, rpcTimeout=980ms 2024-11-13T18:00:05,673 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346., hostname=39e84130bbc9,43723,1731520689798, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=989ms, rpcTimeout=980ms 2024-11-13T18:00:05,673 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346., hostname=39e84130bbc9,43723,1731520689798, seqNum=2 from cache 2024-11-13T18:00:05,686 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=427 (was 447), ProcessCount=11 (was 11), AvailableMemoryMB=1296 (was 1438) 2024-11-13T18:00:05,687 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:00:05,687 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:00:05,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:00:05,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:00:05,693 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:00:05,693 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T18:00:05,694 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:00:05,694 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:00:05,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:00:05,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:00:05,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:00:05,695 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T18:00:05,695 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=148815398, stopped=false 2024-11-13T18:00:05,695 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=39e84130bbc9,35773,1731520688728 2024-11-13T18:00:05,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43723-0x100ed3fe3520001, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T18:00:05,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T18:00:05,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43723-0x100ed3fe3520001, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:00:05,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:00:05,697 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:00:05,698 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:00:05,699 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43723-0x100ed3fe3520001, quorum=127.0.0.1:53564, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:00:05,699 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:00:05,699 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:00:05,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:00:05,699 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '39e84130bbc9,43723,1731520689798' ***** 2024-11-13T18:00:05,700 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T18:00:05,700 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T18:00:05,700 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T18:00:05,700 INFO [RS:0;39e84130bbc9:43723 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T18:00:05,700 INFO [RS:0;39e84130bbc9:43723 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T18:00:05,701 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(3091): Received CLOSE for dc926cd9c7c1f78f2949ef7a52bf7b2d 2024-11-13T18:00:05,702 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(3091): Received CLOSE for 56ff372ec0bdb2b139d60268e9e63346 2024-11-13T18:00:05,702 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(3091): Received CLOSE for 1b7c345630c8c5e80a6529162c393062 2024-11-13T18:00:05,702 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(3091): Received CLOSE for 1d7a32f0d7ece74c8bc407457fff622b 2024-11-13T18:00:05,702 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(3091): Received CLOSE for 43936dcb94433de7b46dc65301fffb7e 2024-11-13T18:00:05,702 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(3091): Received CLOSE for 0a98fe002b6f18a2d07205dfbffd3b04 2024-11-13T18:00:05,702 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(3091): Received CLOSE for 589c6b5058af0a6bb92e99c352ec34b9 2024-11-13T18:00:05,702 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(3091): Received CLOSE for 279f8f9c9faa5090c68e4d3f7af73bb8 2024-11-13T18:00:05,702 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dc926cd9c7c1f78f2949ef7a52bf7b2d, disabling compactions & flushes 2024-11-13T18:00:05,702 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(3091): Received CLOSE for 56e1e10d2d6d43938ddcb116091e6585 2024-11-13T18:00:05,702 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d. 2024-11-13T18:00:05,703 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(3091): Received CLOSE for d83adb705607529a1edc955b54ca2574 2024-11-13T18:00:05,703 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d. 2024-11-13T18:00:05,703 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d. after waiting 0 ms 2024-11-13T18:00:05,703 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(959): stopping server 39e84130bbc9,43723,1731520689798 2024-11-13T18:00:05,703 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d. 2024-11-13T18:00:05,703 INFO [RS:0;39e84130bbc9:43723 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:00:05,703 INFO [RS:0;39e84130bbc9:43723 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;39e84130bbc9:43723. 2024-11-13T18:00:05,703 DEBUG [RS:0;39e84130bbc9:43723 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:00:05,703 DEBUG [RS:0;39e84130bbc9:43723 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:00:05,703 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T18:00:05,703 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T18:00:05,703 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T18:00:05,703 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T18:00:05,704 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-13T18:00:05,704 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T18:00:05,704 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T18:00:05,704 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T18:00:05,704 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T18:00:05,704 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T18:00:05,704 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1351): Waiting on 0a98fe002b6f18a2d07205dfbffd3b04, 1588230740, 1b7c345630c8c5e80a6529162c393062, 1d7a32f0d7ece74c8bc407457fff622b, 279f8f9c9faa5090c68e4d3f7af73bb8, 43936dcb94433de7b46dc65301fffb7e, 56e1e10d2d6d43938ddcb116091e6585, 56ff372ec0bdb2b139d60268e9e63346, 589c6b5058af0a6bb92e99c352ec34b9, d83adb705607529a1edc955b54ca2574, dc926cd9c7c1f78f2949ef7a52bf7b2d 2024-11-13T18:00:05,707 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing dc926cd9c7c1f78f2949ef7a52bf7b2d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-13T18:00:05,707 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-13T18:00:05,781 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnNextAsync/dc926cd9c7c1f78f2949ef7a52bf7b2d/.tmp/testFamily/38da13b30de34de18f29757d6056eb5f is 61, key is row-0/testFamily:testQualifier/1731520762013/Put/seqid=0 2024-11-13T18:00:05,786 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740/.tmp/info/7d67156fb48a4c789432a68e714ca875 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9./info:regioninfo/1731520753225/Put/seqid=0 2024-11-13T18:00:05,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741855_1031 (size=5049) 2024-11-13T18:00:05,799 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnNextAsync/dc926cd9c7c1f78f2949ef7a52bf7b2d/.tmp/testFamily/38da13b30de34de18f29757d6056eb5f 2024-11-13T18:00:05,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741856_1032 (size=20564) 2024-11-13T18:00:05,859 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnNextAsync/dc926cd9c7c1f78f2949ef7a52bf7b2d/.tmp/testFamily/38da13b30de34de18f29757d6056eb5f as hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnNextAsync/dc926cd9c7c1f78f2949ef7a52bf7b2d/testFamily/38da13b30de34de18f29757d6056eb5f 2024-11-13T18:00:05,872 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnNextAsync/dc926cd9c7c1f78f2949ef7a52bf7b2d/testFamily/38da13b30de34de18f29757d6056eb5f, entries=4, sequenceid=8, filesize=4.9 K 2024-11-13T18:00:05,883 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for dc926cd9c7c1f78f2949ef7a52bf7b2d in 175ms, sequenceid=8, compaction requested=false 2024-11-13T18:00:05,897 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnNextAsync/dc926cd9c7c1f78f2949ef7a52bf7b2d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-13T18:00:05,900 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d. 2024-11-13T18:00:05,900 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dc926cd9c7c1f78f2949ef7a52bf7b2d: Waiting for close lock at 1731520805702Running coprocessor pre-close hooks at 1731520805702Disabling compacts and flushes for region at 1731520805702Disabling writes for close at 1731520805703 (+1 ms)Obtaining lock to block concurrent updates at 1731520805707 (+4 ms)Preparing flush snapshotting stores in dc926cd9c7c1f78f2949ef7a52bf7b2d at 1731520805707Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731520805715 (+8 ms)Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d. at 1731520805717 (+2 ms)Flushing dc926cd9c7c1f78f2949ef7a52bf7b2d/testFamily: creating writer at 1731520805719 (+2 ms)Flushing dc926cd9c7c1f78f2949ef7a52bf7b2d/testFamily: appending metadata at 1731520805773 (+54 ms)Flushing dc926cd9c7c1f78f2949ef7a52bf7b2d/testFamily: closing flushed file at 1731520805777 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f8adf0c: reopening flushed file at 1731520805856 (+79 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for dc926cd9c7c1f78f2949ef7a52bf7b2d in 175ms, sequenceid=8, compaction requested=false at 1731520805883 (+27 ms)Writing region close event to WAL at 1731520805890 (+7 ms)Running coprocessor post-close hooks at 1731520805898 (+8 ms)Closed at 1731520805900 (+2 ms) 2024-11-13T18:00:05,901 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731520760989.dc926cd9c7c1f78f2949ef7a52bf7b2d. 2024-11-13T18:00:05,901 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 56ff372ec0bdb2b139d60268e9e63346, disabling compactions & flushes 2024-11-13T18:00:05,901 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346. 2024-11-13T18:00:05,901 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346. 2024-11-13T18:00:05,901 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346. after waiting 0 ms 2024-11-13T18:00:05,901 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346. 2024-11-13T18:00:05,901 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 56ff372ec0bdb2b139d60268e9e63346 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-13T18:00:05,905 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1351): Waiting on 0a98fe002b6f18a2d07205dfbffd3b04, 1588230740, 1b7c345630c8c5e80a6529162c393062, 1d7a32f0d7ece74c8bc407457fff622b, 279f8f9c9faa5090c68e4d3f7af73bb8, 43936dcb94433de7b46dc65301fffb7e, 56e1e10d2d6d43938ddcb116091e6585, 56ff372ec0bdb2b139d60268e9e63346, 589c6b5058af0a6bb92e99c352ec34b9, d83adb705607529a1edc955b54ca2574 2024-11-13T18:00:05,923 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnOpenScannerAsync/56ff372ec0bdb2b139d60268e9e63346/.tmp/testFamily/1445599c7e8f4d5babd9ec512738e0ae is 61, key is row-0/testFamily:testQualifier/1731520802651/Put/seqid=0 2024-11-13T18:00:05,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741857_1033 (size=5049) 2024-11-13T18:00:05,932 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnOpenScannerAsync/56ff372ec0bdb2b139d60268e9e63346/.tmp/testFamily/1445599c7e8f4d5babd9ec512738e0ae 2024-11-13T18:00:05,945 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnOpenScannerAsync/56ff372ec0bdb2b139d60268e9e63346/.tmp/testFamily/1445599c7e8f4d5babd9ec512738e0ae as hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnOpenScannerAsync/56ff372ec0bdb2b139d60268e9e63346/testFamily/1445599c7e8f4d5babd9ec512738e0ae 2024-11-13T18:00:05,956 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnOpenScannerAsync/56ff372ec0bdb2b139d60268e9e63346/testFamily/1445599c7e8f4d5babd9ec512738e0ae, entries=4, sequenceid=8, filesize=4.9 K 2024-11-13T18:00:05,959 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 56ff372ec0bdb2b139d60268e9e63346 in 58ms, sequenceid=8, compaction requested=false 2024-11-13T18:00:05,959 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-13T18:00:05,967 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnOpenScannerAsync/56ff372ec0bdb2b139d60268e9e63346/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-13T18:00:05,968 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346. 2024-11-13T18:00:05,968 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 56ff372ec0bdb2b139d60268e9e63346: Waiting for close lock at 1731520805901Running coprocessor pre-close hooks at 1731520805901Disabling compacts and flushes for region at 1731520805901Disabling writes for close at 1731520805901Obtaining lock to block concurrent updates at 1731520805901Preparing flush snapshotting stores in 56ff372ec0bdb2b139d60268e9e63346 at 1731520805901Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731520805902 (+1 ms)Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346. at 1731520805903 (+1 ms)Flushing 56ff372ec0bdb2b139d60268e9e63346/testFamily: creating writer at 1731520805903Flushing 56ff372ec0bdb2b139d60268e9e63346/testFamily: appending metadata at 1731520805922 (+19 ms)Flushing 56ff372ec0bdb2b139d60268e9e63346/testFamily: closing flushed file at 1731520805923 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7618c3a: reopening flushed file at 1731520805944 (+21 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 56ff372ec0bdb2b139d60268e9e63346 in 58ms, sequenceid=8, compaction requested=false at 1731520805959 (+15 ms)Writing region close event to WAL at 1731520805962 (+3 ms)Running coprocessor post-close hooks at 1731520805968 (+6 ms)Closed at 1731520805968 2024-11-13T18:00:05,969 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731520799597.56ff372ec0bdb2b139d60268e9e63346. 2024-11-13T18:00:05,969 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1b7c345630c8c5e80a6529162c393062, disabling compactions & flushes 2024-11-13T18:00:05,969 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062. 2024-11-13T18:00:05,969 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062. 2024-11-13T18:00:05,969 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062. after waiting 0 ms 2024-11-13T18:00:05,969 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062. 2024-11-13T18:00:05,969 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1b7c345630c8c5e80a6529162c393062 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-13T18:00:06,007 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnNextAsync/1b7c345630c8c5e80a6529162c393062/.tmp/testFamily/67f35c7944db44c69a9c618eba596bc0 is 61, key is row-0/testFamily:testQualifier/1731520774272/Put/seqid=0 2024-11-13T18:00:06,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741858_1034 (size=5049) 2024-11-13T18:00:06,105 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1351): Waiting on 0a98fe002b6f18a2d07205dfbffd3b04, 1588230740, 1b7c345630c8c5e80a6529162c393062, 1d7a32f0d7ece74c8bc407457fff622b, 279f8f9c9faa5090c68e4d3f7af73bb8, 43936dcb94433de7b46dc65301fffb7e, 56e1e10d2d6d43938ddcb116091e6585, 589c6b5058af0a6bb92e99c352ec34b9, d83adb705607529a1edc955b54ca2574 2024-11-13T18:00:06,186 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-13T18:00:06,206 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740/.tmp/info/7d67156fb48a4c789432a68e714ca875 2024-11-13T18:00:06,244 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740/.tmp/ns/6c518a839e944a589b1995361344a542 is 43, key is default/ns:d/1731520752306/Put/seqid=0 2024-11-13T18:00:06,256 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:00:06,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741859_1035 (size=5153) 2024-11-13T18:00:06,260 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740/.tmp/ns/6c518a839e944a589b1995361344a542 2024-11-13T18:00:06,305 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1351): Waiting on 0a98fe002b6f18a2d07205dfbffd3b04, 1588230740, 1b7c345630c8c5e80a6529162c393062, 1d7a32f0d7ece74c8bc407457fff622b, 279f8f9c9faa5090c68e4d3f7af73bb8, 43936dcb94433de7b46dc65301fffb7e, 56e1e10d2d6d43938ddcb116091e6585, 589c6b5058af0a6bb92e99c352ec34b9, d83adb705607529a1edc955b54ca2574 2024-11-13T18:00:06,316 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740/.tmp/table/41b1f2d43f354f8691558fa32653f676 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1731520753246/Put/seqid=0 2024-11-13T18:00:06,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741860_1036 (size=6699) 2024-11-13T18:00:06,324 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740/.tmp/table/41b1f2d43f354f8691558fa32653f676 2024-11-13T18:00:06,336 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740/.tmp/info/7d67156fb48a4c789432a68e714ca875 as hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740/info/7d67156fb48a4c789432a68e714ca875 2024-11-13T18:00:06,346 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740/info/7d67156fb48a4c789432a68e714ca875, entries=100, sequenceid=56, filesize=20.1 K 2024-11-13T18:00:06,348 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740/.tmp/ns/6c518a839e944a589b1995361344a542 as hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740/ns/6c518a839e944a589b1995361344a542 2024-11-13T18:00:06,360 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740/ns/6c518a839e944a589b1995361344a542, entries=2, sequenceid=56, filesize=5.0 K 2024-11-13T18:00:06,364 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740/.tmp/table/41b1f2d43f354f8691558fa32653f676 as hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740/table/41b1f2d43f354f8691558fa32653f676 2024-11-13T18:00:06,384 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740/table/41b1f2d43f354f8691558fa32653f676, entries=20, sequenceid=56, filesize=6.5 K 2024-11-13T18:00:06,386 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 682ms, sequenceid=56, compaction requested=false 2024-11-13T18:00:06,433 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnNextAsync/1b7c345630c8c5e80a6529162c393062/.tmp/testFamily/67f35c7944db44c69a9c618eba596bc0 2024-11-13T18:00:06,437 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-13T18:00:06,439 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T18:00:06,439 INFO [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T18:00:06,439 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731520805704Running coprocessor pre-close hooks at 1731520805704Disabling compacts and flushes for region at 1731520805704Disabling writes for close at 1731520805704Obtaining lock to block concurrent updates at 1731520805707 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1731520805707Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1731520805715 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731520805717 (+2 ms)Flushing 1588230740/info: creating writer at 1731520805720 (+3 ms)Flushing 1588230740/info: appending metadata at 1731520805781 (+61 ms)Flushing 1588230740/info: closing flushed file at 1731520805781Flushing 1588230740/ns: creating writer at 1731520806217 (+436 ms)Flushing 1588230740/ns: appending metadata at 1731520806243 (+26 ms)Flushing 1588230740/ns: closing flushed file at 1731520806243Flushing 1588230740/table: creating writer at 1731520806286 (+43 ms)Flushing 1588230740/table: appending metadata at 1731520806314 (+28 ms)Flushing 1588230740/table: closing flushed file at 1731520806315 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68804e47: reopening flushed file at 1731520806334 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34d57887: reopening flushed file at 1731520806347 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74d54726: reopening flushed file at 1731520806360 (+13 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 682ms, sequenceid=56, compaction requested=false at 1731520806386 (+26 ms)Writing region close event to WAL at 1731520806409 (+23 ms)Running coprocessor post-close hooks at 1731520806438 (+29 ms)Closed at 1731520806439 (+1 ms) 2024-11-13T18:00:06,439 DEBUG [RS_CLOSE_META-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T18:00:06,463 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnNextAsync/1b7c345630c8c5e80a6529162c393062/.tmp/testFamily/67f35c7944db44c69a9c618eba596bc0 as hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnNextAsync/1b7c345630c8c5e80a6529162c393062/testFamily/67f35c7944db44c69a9c618eba596bc0 2024-11-13T18:00:06,476 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnNextAsync/1b7c345630c8c5e80a6529162c393062/testFamily/67f35c7944db44c69a9c618eba596bc0, entries=4, sequenceid=8, filesize=4.9 K 2024-11-13T18:00:06,483 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1b7c345630c8c5e80a6529162c393062 in 514ms, sequenceid=8, compaction requested=false 2024-11-13T18:00:06,494 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnNextAsync/1b7c345630c8c5e80a6529162c393062/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-13T18:00:06,498 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062. 2024-11-13T18:00:06,498 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1b7c345630c8c5e80a6529162c393062: Waiting for close lock at 1731520805969Running coprocessor pre-close hooks at 1731520805969Disabling compacts and flushes for region at 1731520805969Disabling writes for close at 1731520805969Obtaining lock to block concurrent updates at 1731520805969Preparing flush snapshotting stores in 1b7c345630c8c5e80a6529162c393062 at 1731520805969Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731520805969Flushing stores of testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062. at 1731520805970 (+1 ms)Flushing 1b7c345630c8c5e80a6529162c393062/testFamily: creating writer at 1731520805970Flushing 1b7c345630c8c5e80a6529162c393062/testFamily: appending metadata at 1731520806007 (+37 ms)Flushing 1b7c345630c8c5e80a6529162c393062/testFamily: closing flushed file at 1731520806007Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20d26d95: reopening flushed file at 1731520806461 (+454 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1b7c345630c8c5e80a6529162c393062 in 514ms, sequenceid=8, compaction requested=false at 1731520806483 (+22 ms)Writing region close event to WAL at 1731520806484 (+1 ms)Running coprocessor post-close hooks at 1731520806497 (+13 ms)Closed at 1731520806498 (+1 ms) 2024-11-13T18:00:06,498 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1731520773248.1b7c345630c8c5e80a6529162c393062. 2024-11-13T18:00:06,498 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1d7a32f0d7ece74c8bc407457fff622b, disabling compactions & flushes 2024-11-13T18:00:06,498 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b. 2024-11-13T18:00:06,498 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b. 2024-11-13T18:00:06,498 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b. after waiting 0 ms 2024-11-13T18:00:06,498 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b. 2024-11-13T18:00:06,498 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1d7a32f0d7ece74c8bc407457fff622b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-13T18:00:06,506 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1351): Waiting on 0a98fe002b6f18a2d07205dfbffd3b04, 1d7a32f0d7ece74c8bc407457fff622b, 279f8f9c9faa5090c68e4d3f7af73bb8, 43936dcb94433de7b46dc65301fffb7e, 56e1e10d2d6d43938ddcb116091e6585, 589c6b5058af0a6bb92e99c352ec34b9, d83adb705607529a1edc955b54ca2574 2024-11-13T18:00:06,524 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnNext/1d7a32f0d7ece74c8bc407457fff622b/.tmp/testFamily/b47a20e03caf4b7785ff244f016cc6b8 is 61, key is row-0/testFamily:testQualifier/1731520778354/Put/seqid=0 2024-11-13T18:00:06,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741861_1037 (size=5049) 2024-11-13T18:00:06,580 INFO [regionserver/39e84130bbc9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T18:00:06,580 INFO [regionserver/39e84130bbc9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T18:00:06,706 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1321): Waiting on 7 regions to close 2024-11-13T18:00:06,706 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1325): Online Regions={1d7a32f0d7ece74c8bc407457fff622b=testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b., 43936dcb94433de7b46dc65301fffb7e=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e., 0a98fe002b6f18a2d07205dfbffd3b04=hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04., 589c6b5058af0a6bb92e99c352ec34b9=testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9., 279f8f9c9faa5090c68e4d3f7af73bb8=testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8., 56e1e10d2d6d43938ddcb116091e6585=hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585., d83adb705607529a1edc955b54ca2574=testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574.} 2024-11-13T18:00:06,706 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1351): Waiting on 0a98fe002b6f18a2d07205dfbffd3b04, 1d7a32f0d7ece74c8bc407457fff622b, 279f8f9c9faa5090c68e4d3f7af73bb8, 43936dcb94433de7b46dc65301fffb7e, 56e1e10d2d6d43938ddcb116091e6585, 589c6b5058af0a6bb92e99c352ec34b9, d83adb705607529a1edc955b54ca2574 2024-11-13T18:00:06,907 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1351): Waiting on 0a98fe002b6f18a2d07205dfbffd3b04, 1d7a32f0d7ece74c8bc407457fff622b, 279f8f9c9faa5090c68e4d3f7af73bb8, 43936dcb94433de7b46dc65301fffb7e, 56e1e10d2d6d43938ddcb116091e6585, 589c6b5058af0a6bb92e99c352ec34b9, d83adb705607529a1edc955b54ca2574 2024-11-13T18:00:06,932 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnNext/1d7a32f0d7ece74c8bc407457fff622b/.tmp/testFamily/b47a20e03caf4b7785ff244f016cc6b8 2024-11-13T18:00:06,944 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnNext/1d7a32f0d7ece74c8bc407457fff622b/.tmp/testFamily/b47a20e03caf4b7785ff244f016cc6b8 as hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnNext/1d7a32f0d7ece74c8bc407457fff622b/testFamily/b47a20e03caf4b7785ff244f016cc6b8 2024-11-13T18:00:06,955 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnNext/1d7a32f0d7ece74c8bc407457fff622b/testFamily/b47a20e03caf4b7785ff244f016cc6b8, entries=4, sequenceid=8, filesize=4.9 K 2024-11-13T18:00:06,956 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1d7a32f0d7ece74c8bc407457fff622b in 458ms, sequenceid=8, compaction requested=false 2024-11-13T18:00:06,984 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnNext/1d7a32f0d7ece74c8bc407457fff622b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-13T18:00:06,985 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b. 2024-11-13T18:00:06,986 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1d7a32f0d7ece74c8bc407457fff622b: Waiting for close lock at 1731520806498Running coprocessor pre-close hooks at 1731520806498Disabling compacts and flushes for region at 1731520806498Disabling writes for close at 1731520806498Obtaining lock to block concurrent updates at 1731520806498Preparing flush snapshotting stores in 1d7a32f0d7ece74c8bc407457fff622b at 1731520806498Finished memstore snapshotting testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731520806499 (+1 ms)Flushing stores of testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b. at 1731520806499Flushing 1d7a32f0d7ece74c8bc407457fff622b/testFamily: creating writer at 1731520806500 (+1 ms)Flushing 1d7a32f0d7ece74c8bc407457fff622b/testFamily: appending metadata at 1731520806524 (+24 ms)Flushing 1d7a32f0d7ece74c8bc407457fff622b/testFamily: closing flushed file at 1731520806524Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71d0d100: reopening flushed file at 1731520806943 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1d7a32f0d7ece74c8bc407457fff622b in 458ms, sequenceid=8, compaction requested=false at 1731520806956 (+13 ms)Writing region close event to WAL at 1731520806970 (+14 ms)Running coprocessor post-close hooks at 1731520806985 (+15 ms)Closed at 1731520806985 2024-11-13T18:00:06,986 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1731520777320.1d7a32f0d7ece74c8bc407457fff622b. 2024-11-13T18:00:06,986 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 43936dcb94433de7b46dc65301fffb7e, disabling compactions & flushes 2024-11-13T18:00:06,986 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e. 2024-11-13T18:00:06,986 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e. 2024-11-13T18:00:06,986 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e. after waiting 0 ms 2024-11-13T18:00:06,986 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e. 2024-11-13T18:00:06,986 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 43936dcb94433de7b46dc65301fffb7e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-13T18:00:07,022 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/43936dcb94433de7b46dc65301fffb7e/.tmp/testFamily/8d5685ba322d44e79ec101c90a603438 is 61, key is row-0/testFamily:testQualifier/1731520784444/Put/seqid=0 2024-11-13T18:00:07,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741862_1038 (size=5049) 2024-11-13T18:00:07,041 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/43936dcb94433de7b46dc65301fffb7e/.tmp/testFamily/8d5685ba322d44e79ec101c90a603438 2024-11-13T18:00:07,060 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/43936dcb94433de7b46dc65301fffb7e/.tmp/testFamily/8d5685ba322d44e79ec101c90a603438 as hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/43936dcb94433de7b46dc65301fffb7e/testFamily/8d5685ba322d44e79ec101c90a603438 2024-11-13T18:00:07,075 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/43936dcb94433de7b46dc65301fffb7e/testFamily/8d5685ba322d44e79ec101c90a603438, entries=4, sequenceid=8, filesize=4.9 K 2024-11-13T18:00:07,076 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 43936dcb94433de7b46dc65301fffb7e in 90ms, sequenceid=8, compaction requested=false 2024-11-13T18:00:07,089 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/43936dcb94433de7b46dc65301fffb7e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-13T18:00:07,090 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e. 2024-11-13T18:00:07,091 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 43936dcb94433de7b46dc65301fffb7e: Waiting for close lock at 1731520806986Running coprocessor pre-close hooks at 1731520806986Disabling compacts and flushes for region at 1731520806986Disabling writes for close at 1731520806986Obtaining lock to block concurrent updates at 1731520806986Preparing flush snapshotting stores in 43936dcb94433de7b46dc65301fffb7e at 1731520806986Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731520806987 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e. at 1731520806989 (+2 ms)Flushing 43936dcb94433de7b46dc65301fffb7e/testFamily: creating writer at 1731520806989Flushing 43936dcb94433de7b46dc65301fffb7e/testFamily: appending metadata at 1731520807021 (+32 ms)Flushing 43936dcb94433de7b46dc65301fffb7e/testFamily: closing flushed file at 1731520807021Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1aaddb1a: reopening flushed file at 1731520807054 (+33 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 43936dcb94433de7b46dc65301fffb7e in 90ms, sequenceid=8, compaction requested=false at 1731520807076 (+22 ms)Writing region close event to WAL at 1731520807077 (+1 ms)Running coprocessor post-close hooks at 1731520807090 (+13 ms)Closed at 1731520807090 2024-11-13T18:00:07,091 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731520781404.43936dcb94433de7b46dc65301fffb7e. 2024-11-13T18:00:07,091 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0a98fe002b6f18a2d07205dfbffd3b04, disabling compactions & flushes 2024-11-13T18:00:07,091 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04. 2024-11-13T18:00:07,091 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04. 2024-11-13T18:00:07,091 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04. after waiting 0 ms 2024-11-13T18:00:07,091 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04. 2024-11-13T18:00:07,091 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0a98fe002b6f18a2d07205dfbffd3b04 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-13T18:00:07,107 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1351): Waiting on 0a98fe002b6f18a2d07205dfbffd3b04, 279f8f9c9faa5090c68e4d3f7af73bb8, 56e1e10d2d6d43938ddcb116091e6585, 589c6b5058af0a6bb92e99c352ec34b9, d83adb705607529a1edc955b54ca2574 2024-11-13T18:00:07,120 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnOpenScanner/0a98fe002b6f18a2d07205dfbffd3b04/.tmp/testFamily/4d8edca553664f93a2a5b391d5cecbe9 is 61, key is row-0/testFamily:testQualifier/1731520754931/Put/seqid=0 2024-11-13T18:00:07,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741863_1039 (size=5049) 2024-11-13T18:00:07,132 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnOpenScanner/0a98fe002b6f18a2d07205dfbffd3b04/.tmp/testFamily/4d8edca553664f93a2a5b391d5cecbe9 2024-11-13T18:00:07,146 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnOpenScanner/0a98fe002b6f18a2d07205dfbffd3b04/.tmp/testFamily/4d8edca553664f93a2a5b391d5cecbe9 as hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnOpenScanner/0a98fe002b6f18a2d07205dfbffd3b04/testFamily/4d8edca553664f93a2a5b391d5cecbe9 2024-11-13T18:00:07,157 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnOpenScanner/0a98fe002b6f18a2d07205dfbffd3b04/testFamily/4d8edca553664f93a2a5b391d5cecbe9, entries=4, sequenceid=8, filesize=4.9 K 2024-11-13T18:00:07,158 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0a98fe002b6f18a2d07205dfbffd3b04 in 67ms, sequenceid=8, compaction requested=false 2024-11-13T18:00:07,182 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnOpenScanner/0a98fe002b6f18a2d07205dfbffd3b04/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-13T18:00:07,184 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04. 2024-11-13T18:00:07,184 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0a98fe002b6f18a2d07205dfbffd3b04: Waiting for close lock at 1731520807091Running coprocessor pre-close hooks at 1731520807091Disabling compacts and flushes for region at 1731520807091Disabling writes for close at 1731520807091Obtaining lock to block concurrent updates at 1731520807091Preparing flush snapshotting stores in 0a98fe002b6f18a2d07205dfbffd3b04 at 1731520807091Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731520807092 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04. at 1731520807093 (+1 ms)Flushing 0a98fe002b6f18a2d07205dfbffd3b04/testFamily: creating writer at 1731520807093Flushing 0a98fe002b6f18a2d07205dfbffd3b04/testFamily: appending metadata at 1731520807119 (+26 ms)Flushing 0a98fe002b6f18a2d07205dfbffd3b04/testFamily: closing flushed file at 1731520807119Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10e556a9: reopening flushed file at 1731520807145 (+26 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0a98fe002b6f18a2d07205dfbffd3b04 in 67ms, sequenceid=8, compaction requested=false at 1731520807158 (+13 ms)Writing region close event to WAL at 1731520807169 (+11 ms)Running coprocessor post-close hooks at 1731520807184 (+15 ms)Closed at 1731520807184 2024-11-13T18:00:07,185 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731520753903.0a98fe002b6f18a2d07205dfbffd3b04. 2024-11-13T18:00:07,185 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 589c6b5058af0a6bb92e99c352ec34b9, disabling compactions & flushes 2024-11-13T18:00:07,185 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9. 2024-11-13T18:00:07,185 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9. 2024-11-13T18:00:07,185 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9. after waiting 0 ms 2024-11-13T18:00:07,185 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9. 2024-11-13T18:00:07,185 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 589c6b5058af0a6bb92e99c352ec34b9 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-13T18:00:07,212 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testRetryOutOfOrderScannerNextExceptionAsync/589c6b5058af0a6bb92e99c352ec34b9/.tmp/testFamily/84a6b245aac543e59ff14c2d336e38a3 is 61, key is row-0/testFamily:testQualifier/1731520753789/Put/seqid=0 2024-11-13T18:00:07,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741864_1040 (size=5049) 2024-11-13T18:00:07,222 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testRetryOutOfOrderScannerNextExceptionAsync/589c6b5058af0a6bb92e99c352ec34b9/.tmp/testFamily/84a6b245aac543e59ff14c2d336e38a3 2024-11-13T18:00:07,233 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testRetryOutOfOrderScannerNextExceptionAsync/589c6b5058af0a6bb92e99c352ec34b9/.tmp/testFamily/84a6b245aac543e59ff14c2d336e38a3 as hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testRetryOutOfOrderScannerNextExceptionAsync/589c6b5058af0a6bb92e99c352ec34b9/testFamily/84a6b245aac543e59ff14c2d336e38a3 2024-11-13T18:00:07,241 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testRetryOutOfOrderScannerNextExceptionAsync/589c6b5058af0a6bb92e99c352ec34b9/testFamily/84a6b245aac543e59ff14c2d336e38a3, entries=4, sequenceid=8, filesize=4.9 K 2024-11-13T18:00:07,242 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 589c6b5058af0a6bb92e99c352ec34b9 in 57ms, sequenceid=8, compaction requested=false 2024-11-13T18:00:07,247 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testRetryOutOfOrderScannerNextExceptionAsync/589c6b5058af0a6bb92e99c352ec34b9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-13T18:00:07,248 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9. 2024-11-13T18:00:07,248 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 589c6b5058af0a6bb92e99c352ec34b9: Waiting for close lock at 1731520807185Running coprocessor pre-close hooks at 1731520807185Disabling compacts and flushes for region at 1731520807185Disabling writes for close at 1731520807185Obtaining lock to block concurrent updates at 1731520807185Preparing flush snapshotting stores in 589c6b5058af0a6bb92e99c352ec34b9 at 1731520807185Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731520807185Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9. at 1731520807187 (+2 ms)Flushing 589c6b5058af0a6bb92e99c352ec34b9/testFamily: creating writer at 1731520807187Flushing 589c6b5058af0a6bb92e99c352ec34b9/testFamily: appending metadata at 1731520807211 (+24 ms)Flushing 589c6b5058af0a6bb92e99c352ec34b9/testFamily: closing flushed file at 1731520807212 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dfb2604: reopening flushed file at 1731520807232 (+20 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 589c6b5058af0a6bb92e99c352ec34b9 in 57ms, sequenceid=8, compaction requested=false at 1731520807242 (+10 ms)Writing region close event to WAL at 1731520807243 (+1 ms)Running coprocessor post-close hooks at 1731520807248 (+5 ms)Closed at 1731520807248 2024-11-13T18:00:07,248 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731520752680.589c6b5058af0a6bb92e99c352ec34b9. 2024-11-13T18:00:07,249 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 279f8f9c9faa5090c68e4d3f7af73bb8, disabling compactions & flushes 2024-11-13T18:00:07,249 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8. 2024-11-13T18:00:07,249 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8. 2024-11-13T18:00:07,249 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8. after waiting 0 ms 2024-11-13T18:00:07,249 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8. 2024-11-13T18:00:07,249 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 279f8f9c9faa5090c68e4d3f7af73bb8 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-13T18:00:07,273 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testRetryOutOfOrderScannerNextException/279f8f9c9faa5090c68e4d3f7af73bb8/.tmp/testFamily/07e68bfcf052489e872ae9e2905e9dfe is 61, key is row-0/testFamily:testQualifier/1731520769094/Put/seqid=0 2024-11-13T18:00:07,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741865_1041 (size=5049) 2024-11-13T18:00:07,307 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1351): Waiting on 279f8f9c9faa5090c68e4d3f7af73bb8, 56e1e10d2d6d43938ddcb116091e6585, d83adb705607529a1edc955b54ca2574 2024-11-13T18:00:07,508 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1351): Waiting on 279f8f9c9faa5090c68e4d3f7af73bb8, 56e1e10d2d6d43938ddcb116091e6585, d83adb705607529a1edc955b54ca2574 2024-11-13T18:00:07,672 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:00:07,682 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testRetryOutOfOrderScannerNextException/279f8f9c9faa5090c68e4d3f7af73bb8/.tmp/testFamily/07e68bfcf052489e872ae9e2905e9dfe 2024-11-13T18:00:07,692 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testRetryOutOfOrderScannerNextException/279f8f9c9faa5090c68e4d3f7af73bb8/.tmp/testFamily/07e68bfcf052489e872ae9e2905e9dfe as hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testRetryOutOfOrderScannerNextException/279f8f9c9faa5090c68e4d3f7af73bb8/testFamily/07e68bfcf052489e872ae9e2905e9dfe 2024-11-13T18:00:07,702 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testRetryOutOfOrderScannerNextException/279f8f9c9faa5090c68e4d3f7af73bb8/testFamily/07e68bfcf052489e872ae9e2905e9dfe, entries=4, sequenceid=8, filesize=4.9 K 2024-11-13T18:00:07,704 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 279f8f9c9faa5090c68e4d3f7af73bb8 in 455ms, sequenceid=8, compaction requested=false 2024-11-13T18:00:07,708 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-13T18:00:07,708 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1325): Online Regions={279f8f9c9faa5090c68e4d3f7af73bb8=testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8., 56e1e10d2d6d43938ddcb116091e6585=hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585., d83adb705607529a1edc955b54ca2574=testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574.} 2024-11-13T18:00:07,708 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1351): Waiting on 279f8f9c9faa5090c68e4d3f7af73bb8, 56e1e10d2d6d43938ddcb116091e6585, d83adb705607529a1edc955b54ca2574 2024-11-13T18:00:07,715 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testRetryOutOfOrderScannerNextException/279f8f9c9faa5090c68e4d3f7af73bb8/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-13T18:00:07,716 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8. 2024-11-13T18:00:07,716 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 279f8f9c9faa5090c68e4d3f7af73bb8: Waiting for close lock at 1731520807248Running coprocessor pre-close hooks at 1731520807248Disabling compacts and flushes for region at 1731520807248Disabling writes for close at 1731520807249 (+1 ms)Obtaining lock to block concurrent updates at 1731520807249Preparing flush snapshotting stores in 279f8f9c9faa5090c68e4d3f7af73bb8 at 1731520807249Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731520807249Flushing stores of testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8. at 1731520807250 (+1 ms)Flushing 279f8f9c9faa5090c68e4d3f7af73bb8/testFamily: creating writer at 1731520807250Flushing 279f8f9c9faa5090c68e4d3f7af73bb8/testFamily: appending metadata at 1731520807273 (+23 ms)Flushing 279f8f9c9faa5090c68e4d3f7af73bb8/testFamily: closing flushed file at 1731520807273Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@542e5ffa: reopening flushed file at 1731520807691 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 279f8f9c9faa5090c68e4d3f7af73bb8 in 455ms, sequenceid=8, compaction requested=false at 1731520807704 (+13 ms)Writing region close event to WAL at 1731520807709 (+5 ms)Running coprocessor post-close hooks at 1731520807716 (+7 ms)Closed at 1731520807716 2024-11-13T18:00:07,716 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1731520768060.279f8f9c9faa5090c68e4d3f7af73bb8. 2024-11-13T18:00:07,717 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 56e1e10d2d6d43938ddcb116091e6585, disabling compactions & flushes 2024-11-13T18:00:07,717 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585. 2024-11-13T18:00:07,717 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585. 2024-11-13T18:00:07,717 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585. after waiting 0 ms 2024-11-13T18:00:07,717 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585. 2024-11-13T18:00:07,717 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 56e1e10d2d6d43938ddcb116091e6585 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-13T18:00:07,744 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnNext/56e1e10d2d6d43938ddcb116091e6585/.tmp/testFamily/8146d1eafe814ccc931162d282d82005 is 61, key is row-0/testFamily:testQualifier/1731520793544/Put/seqid=0 2024-11-13T18:00:07,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741866_1042 (size=5049) 2024-11-13T18:00:07,751 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnNext/56e1e10d2d6d43938ddcb116091e6585/.tmp/testFamily/8146d1eafe814ccc931162d282d82005 2024-11-13T18:00:07,760 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnNext/56e1e10d2d6d43938ddcb116091e6585/.tmp/testFamily/8146d1eafe814ccc931162d282d82005 as hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnNext/56e1e10d2d6d43938ddcb116091e6585/testFamily/8146d1eafe814ccc931162d282d82005 2024-11-13T18:00:07,769 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnNext/56e1e10d2d6d43938ddcb116091e6585/testFamily/8146d1eafe814ccc931162d282d82005, entries=4, sequenceid=8, filesize=4.9 K 2024-11-13T18:00:07,770 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 56e1e10d2d6d43938ddcb116091e6585 in 53ms, sequenceid=8, compaction requested=false 2024-11-13T18:00:07,775 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/hbase/testMetaScanTimeoutOnNext/56e1e10d2d6d43938ddcb116091e6585/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-13T18:00:07,775 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585. 2024-11-13T18:00:07,775 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 56e1e10d2d6d43938ddcb116091e6585: Waiting for close lock at 1731520807716Running coprocessor pre-close hooks at 1731520807717 (+1 ms)Disabling compacts and flushes for region at 1731520807717Disabling writes for close at 1731520807717Obtaining lock to block concurrent updates at 1731520807717Preparing flush snapshotting stores in 56e1e10d2d6d43938ddcb116091e6585 at 1731520807717Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731520807717Flushing stores of hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585. at 1731520807718 (+1 ms)Flushing 56e1e10d2d6d43938ddcb116091e6585/testFamily: creating writer at 1731520807718Flushing 56e1e10d2d6d43938ddcb116091e6585/testFamily: appending metadata at 1731520807743 (+25 ms)Flushing 56e1e10d2d6d43938ddcb116091e6585/testFamily: closing flushed file at 1731520807743Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f6ab221: reopening flushed file at 1731520807759 (+16 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 56e1e10d2d6d43938ddcb116091e6585 in 53ms, sequenceid=8, compaction requested=false at 1731520807770 (+11 ms)Writing region close event to WAL at 1731520807771 (+1 ms)Running coprocessor post-close hooks at 1731520807775 (+4 ms)Closed at 1731520807775 2024-11-13T18:00:07,776 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1731520790486.56e1e10d2d6d43938ddcb116091e6585. 2024-11-13T18:00:07,776 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d83adb705607529a1edc955b54ca2574, disabling compactions & flushes 2024-11-13T18:00:07,776 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574. 2024-11-13T18:00:07,776 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574. 2024-11-13T18:00:07,776 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574. after waiting 0 ms 2024-11-13T18:00:07,776 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574. 2024-11-13T18:00:07,776 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d83adb705607529a1edc955b54ca2574 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-13T18:00:07,794 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnOpenScanner/d83adb705607529a1edc955b54ca2574/.tmp/testFamily/9847ee424439442a8d4d1c5d61617366 is 61, key is row-0/testFamily:testQualifier/1731520770195/Put/seqid=0 2024-11-13T18:00:07,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741867_1043 (size=5049) 2024-11-13T18:00:07,908 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1351): Waiting on d83adb705607529a1edc955b54ca2574 2024-11-13T18:00:08,109 DEBUG [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1351): Waiting on d83adb705607529a1edc955b54ca2574 2024-11-13T18:00:08,199 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnOpenScanner/d83adb705607529a1edc955b54ca2574/.tmp/testFamily/9847ee424439442a8d4d1c5d61617366 2024-11-13T18:00:08,209 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnOpenScanner/d83adb705607529a1edc955b54ca2574/.tmp/testFamily/9847ee424439442a8d4d1c5d61617366 as hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnOpenScanner/d83adb705607529a1edc955b54ca2574/testFamily/9847ee424439442a8d4d1c5d61617366 2024-11-13T18:00:08,216 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnOpenScanner/d83adb705607529a1edc955b54ca2574/testFamily/9847ee424439442a8d4d1c5d61617366, entries=4, sequenceid=8, filesize=4.9 K 2024-11-13T18:00:08,217 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d83adb705607529a1edc955b54ca2574 in 441ms, sequenceid=8, compaction requested=false 2024-11-13T18:00:08,222 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/data/default/testNormalScanTimeoutOnOpenScanner/d83adb705607529a1edc955b54ca2574/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-13T18:00:08,223 INFO [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574. 2024-11-13T18:00:08,223 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d83adb705607529a1edc955b54ca2574: Waiting for close lock at 1731520807776Running coprocessor pre-close hooks at 1731520807776Disabling compacts and flushes for region at 1731520807776Disabling writes for close at 1731520807776Obtaining lock to block concurrent updates at 1731520807776Preparing flush snapshotting stores in d83adb705607529a1edc955b54ca2574 at 1731520807776Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731520807776Flushing stores of testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574. at 1731520807777 (+1 ms)Flushing d83adb705607529a1edc955b54ca2574/testFamily: creating writer at 1731520807777Flushing d83adb705607529a1edc955b54ca2574/testFamily: appending metadata at 1731520807794 (+17 ms)Flushing d83adb705607529a1edc955b54ca2574/testFamily: closing flushed file at 1731520807794Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6174be40: reopening flushed file at 1731520808207 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d83adb705607529a1edc955b54ca2574 in 441ms, sequenceid=8, compaction requested=false at 1731520808217 (+10 ms)Writing region close event to WAL at 1731520808218 (+1 ms)Running coprocessor post-close hooks at 1731520808223 (+5 ms)Closed at 1731520808223 2024-11-13T18:00:08,223 DEBUG [RS_CLOSE_REGION-regionserver/39e84130bbc9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1731520769166.d83adb705607529a1edc955b54ca2574. 2024-11-13T18:00:08,309 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(976): stopping server 39e84130bbc9,43723,1731520689798; all regions closed. 2024-11-13T18:00:08,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741834_1010 (size=19479) 2024-11-13T18:00:08,322 DEBUG [RS:0;39e84130bbc9:43723 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/oldWALs 2024-11-13T18:00:08,322 INFO [RS:0;39e84130bbc9:43723 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 39e84130bbc9%2C43723%2C1731520689798.meta:.meta(num 1731520752125) 2024-11-13T18:00:08,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741833_1009 (size=20031) 2024-11-13T18:00:08,328 DEBUG [RS:0;39e84130bbc9:43723 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/oldWALs 2024-11-13T18:00:08,328 INFO [RS:0;39e84130bbc9:43723 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 39e84130bbc9%2C43723%2C1731520689798:(num 1731520751678) 2024-11-13T18:00:08,328 DEBUG [RS:0;39e84130bbc9:43723 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:00:08,328 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:00:08,328 INFO [RS:0;39e84130bbc9:43723 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:00:08,328 INFO [RS:0;39e84130bbc9:43723 {}] hbase.ChoreService(370): Chore service for: regionserver/39e84130bbc9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T18:00:08,329 INFO [RS:0;39e84130bbc9:43723 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:00:08,329 INFO [regionserver/39e84130bbc9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:00:08,330 INFO [RS:0;39e84130bbc9:43723 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43723 2024-11-13T18:00:08,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43723-0x100ed3fe3520001, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/39e84130bbc9,43723,1731520689798 2024-11-13T18:00:08,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:00:08,335 INFO [RS:0;39e84130bbc9:43723 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:00:08,338 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [39e84130bbc9,43723,1731520689798] 2024-11-13T18:00:08,339 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/39e84130bbc9,43723,1731520689798 already deleted, retry=false 2024-11-13T18:00:08,340 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 39e84130bbc9,43723,1731520689798 expired; onlineServers=0 2024-11-13T18:00:08,340 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '39e84130bbc9,35773,1731520688728' ***** 2024-11-13T18:00:08,340 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T18:00:08,340 INFO [M:0;39e84130bbc9:35773 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:00:08,340 INFO [M:0;39e84130bbc9:35773 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:00:08,340 DEBUG [M:0;39e84130bbc9:35773 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T18:00:08,340 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T18:00:08,340 DEBUG [M:0;39e84130bbc9:35773 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T18:00:08,340 DEBUG [master/39e84130bbc9:0:becomeActiveMaster-HFileCleaner.large.0-1731520691405 {}] cleaner.HFileCleaner(306): Exit Thread[master/39e84130bbc9:0:becomeActiveMaster-HFileCleaner.large.0-1731520691405,5,FailOnTimeoutGroup] 2024-11-13T18:00:08,341 INFO [M:0;39e84130bbc9:35773 {}] hbase.ChoreService(370): Chore service for: master/39e84130bbc9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T18:00:08,341 DEBUG [master/39e84130bbc9:0:becomeActiveMaster-HFileCleaner.small.0-1731520691406 {}] cleaner.HFileCleaner(306): Exit Thread[master/39e84130bbc9:0:becomeActiveMaster-HFileCleaner.small.0-1731520691406,5,FailOnTimeoutGroup] 2024-11-13T18:00:08,341 INFO [M:0;39e84130bbc9:35773 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:00:08,341 DEBUG [M:0;39e84130bbc9:35773 {}] master.HMaster(1795): Stopping service threads 2024-11-13T18:00:08,341 INFO [M:0;39e84130bbc9:35773 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T18:00:08,341 INFO [M:0;39e84130bbc9:35773 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:00:08,342 INFO [M:0;39e84130bbc9:35773 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T18:00:08,342 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T18:00:08,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T18:00:08,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:00:08,343 DEBUG [M:0;39e84130bbc9:35773 {}] zookeeper.ZKUtil(347): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T18:00:08,343 WARN [M:0;39e84130bbc9:35773 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T18:00:08,344 INFO [M:0;39e84130bbc9:35773 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/.lastflushedseqids 2024-11-13T18:00:08,352 INFO [regionserver/39e84130bbc9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:00:08,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741868_1044 (size=523) 2024-11-13T18:00:08,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43723-0x100ed3fe3520001, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:00:08,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43723-0x100ed3fe3520001, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:00:08,438 INFO [RS:0;39e84130bbc9:43723 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:00:08,439 INFO [RS:0;39e84130bbc9:43723 {}] regionserver.HRegionServer(1031): Exiting; stopping=39e84130bbc9,43723,1731520689798; zookeeper connection closed. 2024-11-13T18:00:08,439 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@32024618 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@32024618 2024-11-13T18:00:08,440 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-13T18:00:08,760 INFO [M:0;39e84130bbc9:35773 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T18:00:08,760 INFO [M:0;39e84130bbc9:35773 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T18:00:08,760 DEBUG [M:0;39e84130bbc9:35773 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T18:00:08,760 INFO [M:0;39e84130bbc9:35773 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:00:08,760 DEBUG [M:0;39e84130bbc9:35773 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:00:08,760 DEBUG [M:0;39e84130bbc9:35773 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T18:00:08,761 DEBUG [M:0;39e84130bbc9:35773 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:00:08,761 INFO [M:0;39e84130bbc9:35773 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=162.90 KB heapSize=190.55 KB 2024-11-13T18:00:08,783 DEBUG [M:0;39e84130bbc9:35773 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/02a861434618455eacb0ee61751d2ec5 is 82, key is hbase:meta,,1/info:regioninfo/1731520752212/Put/seqid=0 2024-11-13T18:00:08,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741869_1045 (size=5672) 2024-11-13T18:00:09,196 INFO [M:0;39e84130bbc9:35773 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/02a861434618455eacb0ee61751d2ec5 2024-11-13T18:00:09,234 DEBUG [M:0;39e84130bbc9:35773 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ecea280b5f434c81afa1547ffc94a0e9 is 811, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731520753253/Put/seqid=0 2024-11-13T18:00:09,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741870_1046 (size=14073) 2024-11-13T18:00:09,259 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-13T18:00:09,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-13T18:00:09,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T18:00:09,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T18:00:09,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-13T18:00:09,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-13T18:00:09,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-13T18:00:09,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-13T18:00:09,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-13T18:00:09,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-13T18:00:09,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-13T18:00:09,646 INFO [M:0;39e84130bbc9:35773 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.35 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ecea280b5f434c81afa1547ffc94a0e9 2024-11-13T18:00:09,654 INFO [M:0;39e84130bbc9:35773 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ecea280b5f434c81afa1547ffc94a0e9 2024-11-13T18:00:09,678 DEBUG [M:0;39e84130bbc9:35773 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8031cbfed84f4f3e8ad1ef24cd87310f is 69, key is 39e84130bbc9,43723,1731520689798/rs:state/1731520721423/Put/seqid=0 2024-11-13T18:00:09,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741871_1047 (size=5156) 2024-11-13T18:00:10,084 INFO [M:0;39e84130bbc9:35773 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8031cbfed84f4f3e8ad1ef24cd87310f 2024-11-13T18:00:10,091 DEBUG [M:0;39e84130bbc9:35773 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/02a861434618455eacb0ee61751d2ec5 as hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/02a861434618455eacb0ee61751d2ec5 2024-11-13T18:00:10,097 INFO [M:0;39e84130bbc9:35773 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/02a861434618455eacb0ee61751d2ec5, entries=8, sequenceid=313, filesize=5.5 K 2024-11-13T18:00:10,098 DEBUG [M:0;39e84130bbc9:35773 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ecea280b5f434c81afa1547ffc94a0e9 as hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ecea280b5f434c81afa1547ffc94a0e9 2024-11-13T18:00:10,104 INFO [M:0;39e84130bbc9:35773 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ecea280b5f434c81afa1547ffc94a0e9 2024-11-13T18:00:10,105 INFO [M:0;39e84130bbc9:35773 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ecea280b5f434c81afa1547ffc94a0e9, entries=33, sequenceid=313, filesize=13.7 K 2024-11-13T18:00:10,106 DEBUG [M:0;39e84130bbc9:35773 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8031cbfed84f4f3e8ad1ef24cd87310f as hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8031cbfed84f4f3e8ad1ef24cd87310f 2024-11-13T18:00:10,113 INFO [M:0;39e84130bbc9:35773 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33935/user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8031cbfed84f4f3e8ad1ef24cd87310f, entries=1, sequenceid=313, filesize=5.0 K 2024-11-13T18:00:10,115 INFO [M:0;39e84130bbc9:35773 {}] regionserver.HRegion(3140): Finished flush of dataSize ~162.90 KB/166811, heapSize ~190.26 KB/194824, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1354ms, sequenceid=313, compaction requested=false 2024-11-13T18:00:10,117 INFO [M:0;39e84130bbc9:35773 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:00:10,117 DEBUG [M:0;39e84130bbc9:35773 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731520808760Disabling compacts and flushes for region at 1731520808760Disabling writes for close at 1731520808760Obtaining lock to block concurrent updates at 1731520808761 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731520808761Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166811, getHeapSize=195064, getOffHeapSize=0, getCellsCount=360 at 1731520808761Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731520808762 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731520808762Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731520808783 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731520808783Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731520809204 (+421 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731520809232 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731520809232Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731520809654 (+422 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731520809677 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731520809677Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3caf921d: reopening flushed file at 1731520810090 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44ff08d9: reopening flushed file at 1731520810097 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f1b3b11: reopening flushed file at 1731520810105 (+8 ms)Finished flush of dataSize ~162.90 KB/166811, heapSize ~190.26 KB/194824, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1354ms, sequenceid=313, compaction requested=false at 1731520810115 (+10 ms)Writing region close event to WAL at 1731520810117 (+2 ms)Closed at 1731520810117 2024-11-13T18:00:10,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39637 is added to blk_1073741830_1006 (size=187763) 2024-11-13T18:00:10,121 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/44229257-1a48-d7f9-2b0f-5c5ef5c027f0/MasterData/WALs/39e84130bbc9,35773,1731520688728/39e84130bbc9%2C35773%2C1731520688728.1731520690589 not finished, retry = 0 2024-11-13T18:00:10,223 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:00:10,223 INFO [M:0;39e84130bbc9:35773 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T18:00:10,223 INFO [M:0;39e84130bbc9:35773 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35773 2024-11-13T18:00:10,223 INFO [M:0;39e84130bbc9:35773 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:00:10,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:00:10,326 INFO [M:0;39e84130bbc9:35773 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:00:10,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35773-0x100ed3fe3520000, quorum=127.0.0.1:53564, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:00:10,334 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@16d0bc3d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T18:00:10,340 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2e1cc352{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T18:00:10,341 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T18:00:10,341 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a781d84{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T18:00:10,341 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@23c598c8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/hadoop.log.dir/,STOPPED} 2024-11-13T18:00:10,351 WARN [BP-2066668531-172.17.0.3-1731520685052 heartbeating to localhost/127.0.0.1:33935 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T18:00:10,351 WARN [BP-2066668531-172.17.0.3-1731520685052 heartbeating to localhost/127.0.0.1:33935 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2066668531-172.17.0.3-1731520685052 (Datanode Uuid a1c690e6-c995-4d71-94ea-97556a439951) service to localhost/127.0.0.1:33935 2024-11-13T18:00:10,357 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T18:00:10,358 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T18:00:10,359 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/cluster_97dbc57c-85a5-456c-2cc5-cce186832a72/data/data1/current/BP-2066668531-172.17.0.3-1731520685052 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T18:00:10,359 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/cluster_97dbc57c-85a5-456c-2cc5-cce186832a72/data/data2/current/BP-2066668531-172.17.0.3-1731520685052 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T18:00:10,360 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T18:00:10,383 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4df87c21{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T18:00:10,384 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@29047fa7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T18:00:10,384 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T18:00:10,385 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5c1e5f71{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T18:00:10,385 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3653a146{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f7e3034c-7aa9-b533-ad95-173a995ade2e/hadoop.log.dir/,STOPPED} 2024-11-13T18:00:10,401 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T18:00:10,432 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down