2024-12-04 02:33:03,293 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-04 02:33:03,322 main DEBUG Took 0.027374 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-04 02:33:03,322 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-04 02:33:03,323 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-04 02:33:03,324 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-04 02:33:03,326 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 02:33:03,333 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-04 02:33:03,349 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 02:33:03,350 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 02:33:03,351 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 02:33:03,352 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 02:33:03,353 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 02:33:03,353 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 02:33:03,355 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 02:33:03,355 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 02:33:03,356 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 02:33:03,357 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 02:33:03,357 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 02:33:03,358 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 02:33:03,359 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 02:33:03,359 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 02:33:03,360 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 02:33:03,360 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 02:33:03,361 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 02:33:03,361 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 02:33:03,362 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 02:33:03,363 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 02:33:03,364 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 02:33:03,364 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 02:33:03,365 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 02:33:03,365 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 02:33:03,366 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 02:33:03,366 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-04 02:33:03,368 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 02:33:03,370 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-04 02:33:03,373 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-04 02:33:03,373 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-04 02:33:03,375 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-04 02:33:03,376 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-04 02:33:03,383 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-04 02:33:03,386 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-04 02:33:03,388 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-04 02:33:03,388 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-04 02:33:03,388 main DEBUG createAppenders(={Console}) 2024-12-04 02:33:03,389 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-12-04 02:33:03,389 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-04 02:33:03,390 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-12-04 02:33:03,390 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-04 02:33:03,391 main DEBUG OutputStream closed 2024-12-04 02:33:03,391 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-04 02:33:03,391 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-04 02:33:03,391 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-12-04 02:33:03,465 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-04 02:33:03,468 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-04 02:33:03,469 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-04 02:33:03,470 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-04 02:33:03,471 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-04 02:33:03,471 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-04 02:33:03,471 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-04 02:33:03,472 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-04 02:33:03,472 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-04 02:33:03,472 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-04 02:33:03,473 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-04 02:33:03,473 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-04 02:33:03,474 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-04 02:33:03,474 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-04 02:33:03,474 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-04 02:33:03,475 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-04 02:33:03,475 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-04 02:33:03,476 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-04 02:33:03,477 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04 02:33:03,478 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-12-04 02:33:03,478 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-04 02:33:03,479 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-12-04T02:33:03,495 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-12-04 02:33:03,499 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-04 02:33:03,499 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04T02:33:03,817 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e 2024-12-04T02:33:03,854 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T02:33:03,875 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/cluster_94449acd-a2fa-e53b-a6f8-948a65223f61, deleteOnExit=true 2024-12-04T02:33:03,875 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T02:33:03,882 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/test.cache.data in system properties and HBase conf 2024-12-04T02:33:03,883 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T02:33:03,884 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/hadoop.log.dir in system properties and HBase conf 2024-12-04T02:33:03,884 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T02:33:03,885 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T02:33:03,886 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T02:33:04,019 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-04T02:33:04,131 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T02:33:04,136 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T02:33:04,136 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T02:33:04,137 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T02:33:04,137 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T02:33:04,138 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T02:33:04,139 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T02:33:04,139 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T02:33:04,140 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T02:33:04,140 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T02:33:04,141 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/nfs.dump.dir in system properties and HBase conf 2024-12-04T02:33:04,141 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/java.io.tmpdir in system properties and HBase conf 2024-12-04T02:33:04,142 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T02:33:04,142 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T02:33:04,143 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T02:33:05,340 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-04T02:33:05,475 INFO [Time-limited test {}] log.Log(170): Logging initialized @2984ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-04T02:33:05,581 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T02:33:05,689 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T02:33:05,727 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T02:33:05,727 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T02:33:05,729 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T02:33:05,748 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T02:33:05,758 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3bcd68eb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/hadoop.log.dir/,AVAILABLE} 2024-12-04T02:33:05,759 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3959e81b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T02:33:05,993 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3ce6ffde{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/java.io.tmpdir/jetty-localhost-45935-hadoop-hdfs-3_4_1-tests_jar-_-any-3518433706240560809/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T02:33:06,003 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5c90255f{HTTP/1.1, (http/1.1)}{localhost:45935} 2024-12-04T02:33:06,004 INFO [Time-limited test {}] server.Server(415): Started @3514ms 2024-12-04T02:33:06,737 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T02:33:06,749 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T02:33:06,754 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T02:33:06,754 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T02:33:06,755 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T02:33:06,756 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f7444f6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/hadoop.log.dir/,AVAILABLE} 2024-12-04T02:33:06,756 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@30fd9473{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T02:33:06,858 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5bc105ee{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/java.io.tmpdir/jetty-localhost-35097-hadoop-hdfs-3_4_1-tests_jar-_-any-8015848244404913526/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T02:33:06,859 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@360caa46{HTTP/1.1, (http/1.1)}{localhost:35097} 2024-12-04T02:33:06,860 INFO [Time-limited test {}] server.Server(415): Started @4371ms 2024-12-04T02:33:06,927 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T02:33:08,249 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/cluster_94449acd-a2fa-e53b-a6f8-948a65223f61/data/data1/current/BP-2088324271-172.17.0.3-1733279584749/current, will proceed with Du for space computation calculation, 2024-12-04T02:33:08,249 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/cluster_94449acd-a2fa-e53b-a6f8-948a65223f61/data/data2/current/BP-2088324271-172.17.0.3-1733279584749/current, will proceed with Du for space computation calculation, 2024-12-04T02:33:08,280 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T02:33:08,330 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf52c58f4525832d8 with lease ID 0xcf07274f312d2c5c: Processing first storage report for DS-3967ebea-8971-42e1-aea7-2a8d7c5a9800 from datanode DatanodeRegistration(127.0.0.1:33875, datanodeUuid=88c4e69d-6a95-4d0f-b6bb-d93f38f96390, infoPort=44099, infoSecurePort=0, ipcPort=38009, storageInfo=lv=-57;cid=testClusterID;nsid=1271945082;c=1733279584749) 2024-12-04T02:33:08,331 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf52c58f4525832d8 with lease ID 0xcf07274f312d2c5c: from storage DS-3967ebea-8971-42e1-aea7-2a8d7c5a9800 node DatanodeRegistration(127.0.0.1:33875, datanodeUuid=88c4e69d-6a95-4d0f-b6bb-d93f38f96390, infoPort=44099, infoSecurePort=0, ipcPort=38009, storageInfo=lv=-57;cid=testClusterID;nsid=1271945082;c=1733279584749), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-04T02:33:08,331 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf52c58f4525832d8 with lease ID 0xcf07274f312d2c5c: Processing first storage report for DS-fd7f481d-e411-42f7-974e-0bb14fc46afb from datanode DatanodeRegistration(127.0.0.1:33875, datanodeUuid=88c4e69d-6a95-4d0f-b6bb-d93f38f96390, infoPort=44099, infoSecurePort=0, ipcPort=38009, storageInfo=lv=-57;cid=testClusterID;nsid=1271945082;c=1733279584749) 2024-12-04T02:33:08,332 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf52c58f4525832d8 with lease ID 0xcf07274f312d2c5c: from storage DS-fd7f481d-e411-42f7-974e-0bb14fc46afb node DatanodeRegistration(127.0.0.1:33875, datanodeUuid=88c4e69d-6a95-4d0f-b6bb-d93f38f96390, infoPort=44099, infoSecurePort=0, ipcPort=38009, storageInfo=lv=-57;cid=testClusterID;nsid=1271945082;c=1733279584749), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T02:33:08,395 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e 2024-12-04T02:33:08,515 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/cluster_94449acd-a2fa-e53b-a6f8-948a65223f61/zookeeper_0, clientPort=53960, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/cluster_94449acd-a2fa-e53b-a6f8-948a65223f61/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/cluster_94449acd-a2fa-e53b-a6f8-948a65223f61/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T02:33:08,531 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53960 2024-12-04T02:33:08,550 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T02:33:08,556 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T02:33:08,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741825_1001 (size=7) 2024-12-04T02:33:09,267 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7 with version=8 2024-12-04T02:33:09,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/hbase-staging 2024-12-04T02:33:09,389 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-04T02:33:09,704 INFO [Time-limited test {}] client.ConnectionUtils(128): master/40249addbad9:0 server-side Connection retries=9 2024-12-04T02:33:09,716 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T02:33:09,716 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T02:33:09,723 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T02:33:09,723 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T02:33:09,723 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T02:33:09,849 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T02:33:09,903 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-04T02:33:09,911 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-04T02:33:09,914 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T02:33:09,935 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 99801 (auto-detected) 2024-12-04T02:33:09,936 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-04T02:33:09,953 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43601 2024-12-04T02:33:09,973 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43601 connecting to ZooKeeper ensemble=127.0.0.1:53960 2024-12-04T02:33:10,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:436010x0, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T02:33:10,065 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43601-0x1019e2f9c250000 connected 2024-12-04T02:33:10,148 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T02:33:10,150 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T02:33:10,165 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T02:33:10,170 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7, hbase.cluster.distributed=false 2024-12-04T02:33:10,199 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T02:33:10,212 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43601 2024-12-04T02:33:10,213 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43601 2024-12-04T02:33:10,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43601 2024-12-04T02:33:10,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43601 2024-12-04T02:33:10,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43601 2024-12-04T02:33:10,348 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/40249addbad9:0 server-side Connection retries=9 2024-12-04T02:33:10,350 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T02:33:10,351 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T02:33:10,351 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T02:33:10,351 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T02:33:10,352 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T02:33:10,356 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T02:33:10,359 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T02:33:10,360 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43941 2024-12-04T02:33:10,363 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43941 connecting to ZooKeeper ensemble=127.0.0.1:53960 2024-12-04T02:33:10,365 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T02:33:10,370 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T02:33:10,404 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:439410x0, quorum=127.0.0.1:53960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T02:33:10,404 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:439410x0, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T02:33:10,405 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43941-0x1019e2f9c250001 connected 2024-12-04T02:33:10,410 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T02:33:10,422 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T02:33:10,426 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43941-0x1019e2f9c250001, quorum=127.0.0.1:53960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T02:33:10,436 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43941-0x1019e2f9c250001, quorum=127.0.0.1:53960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T02:33:10,444 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43941 2024-12-04T02:33:10,445 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43941 2024-12-04T02:33:10,445 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43941 2024-12-04T02:33:10,446 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43941 2024-12-04T02:33:10,446 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43941 2024-12-04T02:33:10,467 DEBUG [M:0;40249addbad9:43601 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;40249addbad9:43601 2024-12-04T02:33:10,468 INFO [master/40249addbad9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/40249addbad9,43601,1733279589466 2024-12-04T02:33:10,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43941-0x1019e2f9c250001, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T02:33:10,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T02:33:10,488 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/40249addbad9,43601,1733279589466 2024-12-04T02:33:10,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T02:33:10,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43941-0x1019e2f9c250001, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T02:33:10,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43941-0x1019e2f9c250001, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T02:33:10,526 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T02:33:10,528 INFO [master/40249addbad9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/40249addbad9,43601,1733279589466 from backup master directory 2024-12-04T02:33:10,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/40249addbad9,43601,1733279589466 2024-12-04T02:33:10,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T02:33:10,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43941-0x1019e2f9c250001, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T02:33:10,535 WARN [master/40249addbad9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T02:33:10,536 INFO [master/40249addbad9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=40249addbad9,43601,1733279589466 2024-12-04T02:33:10,541 INFO [master/40249addbad9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-04T02:33:10,544 INFO [master/40249addbad9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-04T02:33:10,613 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/hbase.id] with ID: 38a6c2e2-1cbb-4126-b063-575290e89383 2024-12-04T02:33:10,613 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/.tmp/hbase.id 2024-12-04T02:33:10,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741826_1002 (size=42) 2024-12-04T02:33:11,026 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/.tmp/hbase.id]:[hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/hbase.id] 2024-12-04T02:33:11,084 INFO [master/40249addbad9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T02:33:11,089 INFO [master/40249addbad9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T02:33:11,109 INFO [master/40249addbad9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-12-04T02:33:11,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T02:33:11,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43941-0x1019e2f9c250001, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T02:33:11,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741827_1003 (size=196) 2024-12-04T02:33:11,572 INFO [master/40249addbad9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T02:33:11,576 INFO [master/40249addbad9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T02:33:11,595 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T02:33:11,600 INFO [master/40249addbad9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T02:33:11,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741828_1004 (size=1189) 2024-12-04T02:33:12,051 INFO [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/data/master/store 2024-12-04T02:33:12,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741829_1005 (size=34) 2024-12-04T02:33:12,485 INFO [master/40249addbad9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-04T02:33:12,489 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:33:12,490 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T02:33:12,490 INFO [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T02:33:12,491 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T02:33:12,492 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T02:33:12,492 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T02:33:12,492 INFO [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T02:33:12,493 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733279592490Disabling compacts and flushes for region at 1733279592490Disabling writes for close at 1733279592492 (+2 ms)Writing region close event to WAL at 1733279592492Closed at 1733279592492 2024-12-04T02:33:12,495 WARN [master/40249addbad9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/data/master/store/.initializing 2024-12-04T02:33:12,496 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/WALs/40249addbad9,43601,1733279589466 2024-12-04T02:33:12,503 INFO [master/40249addbad9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T02:33:12,516 INFO [master/40249addbad9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=40249addbad9%2C43601%2C1733279589466, suffix=, logDir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/WALs/40249addbad9,43601,1733279589466, archiveDir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/oldWALs, maxLogs=10 2024-12-04T02:33:12,536 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/WALs/40249addbad9,43601,1733279589466/40249addbad9%2C43601%2C1733279589466.1733279592520, exclude list is [], retry=0 2024-12-04T02:33:12,550 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33875,DS-3967ebea-8971-42e1-aea7-2a8d7c5a9800,DISK] 2024-12-04T02:33:12,553 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-04T02:33:12,588 INFO [master/40249addbad9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/WALs/40249addbad9,43601,1733279589466/40249addbad9%2C43601%2C1733279589466.1733279592520 2024-12-04T02:33:12,589 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44099:44099)] 2024-12-04T02:33:12,590 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T02:33:12,590 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:33:12,594 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T02:33:12,595 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T02:33:12,636 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T02:33:12,661 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T02:33:12,664 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:33:12,667 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T02:33:12,667 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T02:33:12,671 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T02:33:12,672 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:33:12,673 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T02:33:12,673 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T02:33:12,676 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T02:33:12,677 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:33:12,678 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T02:33:12,678 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T02:33:12,681 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T02:33:12,681 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:33:12,683 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T02:33:12,683 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T02:33:12,687 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T02:33:12,688 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T02:33:12,693 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T02:33:12,693 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T02:33:12,697 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T02:33:12,701 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T02:33:12,706 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T02:33:12,707 INFO [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60467880, jitterRate=-0.09895837306976318}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T02:33:12,713 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733279592609Initializing all the Stores at 1733279592612 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733279592612Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733279592613 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733279592614 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733279592614Cleaning up temporary data from old regions at 1733279592693 (+79 ms)Region opened successfully at 1733279592713 (+20 ms) 2024-12-04T02:33:12,715 INFO [master/40249addbad9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T02:33:12,746 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@786d0c2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=40249addbad9/172.17.0.3:0 2024-12-04T02:33:12,780 INFO [master/40249addbad9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T02:33:12,791 INFO [master/40249addbad9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T02:33:12,791 INFO [master/40249addbad9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T02:33:12,793 INFO [master/40249addbad9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T02:33:12,794 INFO [master/40249addbad9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-04T02:33:12,798 INFO [master/40249addbad9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-04T02:33:12,798 INFO [master/40249addbad9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T02:33:12,819 INFO [master/40249addbad9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T02:33:12,826 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T02:33:12,839 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T02:33:12,842 INFO [master/40249addbad9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T02:33:12,843 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T02:33:12,850 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T02:33:12,852 INFO [master/40249addbad9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T02:33:12,854 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T02:33:12,860 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T02:33:12,862 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T02:33:12,871 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T02:33:12,892 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T02:33:12,902 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T02:33:12,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T02:33:12,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43941-0x1019e2f9c250001, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T02:33:12,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T02:33:12,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43941-0x1019e2f9c250001, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T02:33:12,918 INFO [master/40249addbad9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=40249addbad9,43601,1733279589466, sessionid=0x1019e2f9c250000, setting cluster-up flag (Was=false) 2024-12-04T02:33:12,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43941-0x1019e2f9c250001, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T02:33:12,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T02:33:12,977 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T02:33:12,983 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=40249addbad9,43601,1733279589466 2024-12-04T02:33:13,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43941-0x1019e2f9c250001, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T02:33:13,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T02:33:13,040 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T02:33:13,044 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=40249addbad9,43601,1733279589466 2024-12-04T02:33:13,053 INFO [master/40249addbad9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T02:33:13,116 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T02:33:13,125 INFO [master/40249addbad9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T02:33:13,131 INFO [master/40249addbad9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T02:33:13,137 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 40249addbad9,43601,1733279589466 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T02:33:13,146 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/40249addbad9:0, corePoolSize=5, maxPoolSize=5 2024-12-04T02:33:13,146 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/40249addbad9:0, corePoolSize=5, maxPoolSize=5 2024-12-04T02:33:13,146 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/40249addbad9:0, corePoolSize=5, maxPoolSize=5 2024-12-04T02:33:13,147 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/40249addbad9:0, corePoolSize=5, maxPoolSize=5 2024-12-04T02:33:13,147 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/40249addbad9:0, corePoolSize=10, maxPoolSize=10 2024-12-04T02:33:13,147 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/40249addbad9:0, corePoolSize=1, maxPoolSize=1 2024-12-04T02:33:13,147 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/40249addbad9:0, corePoolSize=2, maxPoolSize=2 2024-12-04T02:33:13,147 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/40249addbad9:0, corePoolSize=1, maxPoolSize=1 2024-12-04T02:33:13,153 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T02:33:13,154 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T02:33:13,157 INFO [master/40249addbad9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733279623157 2024-12-04T02:33:13,158 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(746): ClusterId : 38a6c2e2-1cbb-4126-b063-575290e89383 2024-12-04T02:33:13,159 INFO [master/40249addbad9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T02:33:13,160 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:33:13,161 INFO [master/40249addbad9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T02:33:13,161 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T02:33:13,161 DEBUG [RS:0;40249addbad9:43941 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T02:33:13,165 INFO [master/40249addbad9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T02:33:13,166 INFO [master/40249addbad9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T02:33:13,166 INFO [master/40249addbad9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T02:33:13,166 INFO [master/40249addbad9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T02:33:13,172 INFO [master/40249addbad9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:13,190 DEBUG [RS:0;40249addbad9:43941 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T02:33:13,190 DEBUG [RS:0;40249addbad9:43941 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T02:33:13,191 INFO [master/40249addbad9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T02:33:13,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741831_1007 (size=1321) 2024-12-04T02:33:13,193 INFO [master/40249addbad9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T02:33:13,194 INFO [master/40249addbad9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T02:33:13,197 INFO [master/40249addbad9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T02:33:13,198 INFO [master/40249addbad9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T02:33:13,199 DEBUG [RS:0;40249addbad9:43941 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T02:33:13,200 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/40249addbad9:0:becomeActiveMaster-HFileCleaner.large.0-1733279593199,5,FailOnTimeoutGroup] 2024-12-04T02:33:13,200 DEBUG [RS:0;40249addbad9:43941 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@642ac94e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=40249addbad9/172.17.0.3:0 2024-12-04T02:33:13,201 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/40249addbad9:0:becomeActiveMaster-HFileCleaner.small.0-1733279593200,5,FailOnTimeoutGroup] 2024-12-04T02:33:13,201 INFO [master/40249addbad9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:13,201 INFO [master/40249addbad9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T02:33:13,203 INFO [master/40249addbad9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:13,204 INFO [master/40249addbad9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:13,217 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;40249addbad9:43941 2024-12-04T02:33:13,220 INFO [RS:0;40249addbad9:43941 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T02:33:13,221 INFO [RS:0;40249addbad9:43941 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T02:33:13,221 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T02:33:13,223 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(2659): reportForDuty to master=40249addbad9,43601,1733279589466 with port=43941, startcode=1733279590307 2024-12-04T02:33:13,236 DEBUG [RS:0;40249addbad9:43941 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T02:33:13,265 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43189, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T02:33:13,270 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43601 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 40249addbad9,43941,1733279590307 2024-12-04T02:33:13,272 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43601 {}] master.ServerManager(517): Registering regionserver=40249addbad9,43941,1733279590307 2024-12-04T02:33:13,284 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7 2024-12-04T02:33:13,284 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34037 2024-12-04T02:33:13,284 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T02:33:13,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T02:33:13,293 DEBUG [RS:0;40249addbad9:43941 {}] zookeeper.ZKUtil(111): regionserver:43941-0x1019e2f9c250001, quorum=127.0.0.1:53960, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/40249addbad9,43941,1733279590307 2024-12-04T02:33:13,293 WARN [RS:0;40249addbad9:43941 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T02:33:13,293 INFO [RS:0;40249addbad9:43941 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T02:33:13,293 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/WALs/40249addbad9,43941,1733279590307 2024-12-04T02:33:13,295 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [40249addbad9,43941,1733279590307] 2024-12-04T02:33:13,318 INFO [RS:0;40249addbad9:43941 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T02:33:13,331 INFO [RS:0;40249addbad9:43941 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T02:33:13,336 INFO [RS:0;40249addbad9:43941 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T02:33:13,336 INFO [RS:0;40249addbad9:43941 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:13,337 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T02:33:13,342 INFO [RS:0;40249addbad9:43941 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T02:33:13,343 INFO [RS:0;40249addbad9:43941 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:13,343 DEBUG [RS:0;40249addbad9:43941 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/40249addbad9:0, corePoolSize=1, maxPoolSize=1 2024-12-04T02:33:13,343 DEBUG [RS:0;40249addbad9:43941 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/40249addbad9:0, corePoolSize=1, maxPoolSize=1 2024-12-04T02:33:13,343 DEBUG [RS:0;40249addbad9:43941 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0, corePoolSize=1, maxPoolSize=1 2024-12-04T02:33:13,344 DEBUG [RS:0;40249addbad9:43941 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/40249addbad9:0, corePoolSize=1, maxPoolSize=1 2024-12-04T02:33:13,344 DEBUG [RS:0;40249addbad9:43941 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/40249addbad9:0, corePoolSize=1, maxPoolSize=1 2024-12-04T02:33:13,344 DEBUG [RS:0;40249addbad9:43941 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/40249addbad9:0, corePoolSize=2, maxPoolSize=2 2024-12-04T02:33:13,344 DEBUG [RS:0;40249addbad9:43941 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/40249addbad9:0, corePoolSize=1, maxPoolSize=1 2024-12-04T02:33:13,344 DEBUG [RS:0;40249addbad9:43941 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/40249addbad9:0, corePoolSize=1, maxPoolSize=1 2024-12-04T02:33:13,344 DEBUG [RS:0;40249addbad9:43941 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/40249addbad9:0, corePoolSize=1, maxPoolSize=1 2024-12-04T02:33:13,345 DEBUG [RS:0;40249addbad9:43941 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/40249addbad9:0, corePoolSize=1, maxPoolSize=1 2024-12-04T02:33:13,345 DEBUG [RS:0;40249addbad9:43941 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/40249addbad9:0, corePoolSize=1, maxPoolSize=1 2024-12-04T02:33:13,345 DEBUG [RS:0;40249addbad9:43941 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/40249addbad9:0, corePoolSize=1, maxPoolSize=1 2024-12-04T02:33:13,345 DEBUG [RS:0;40249addbad9:43941 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/40249addbad9:0, corePoolSize=3, maxPoolSize=3 2024-12-04T02:33:13,345 DEBUG [RS:0;40249addbad9:43941 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/40249addbad9:0, corePoolSize=3, maxPoolSize=3 2024-12-04T02:33:13,346 INFO [RS:0;40249addbad9:43941 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:13,347 INFO [RS:0;40249addbad9:43941 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:13,347 INFO [RS:0;40249addbad9:43941 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:13,347 INFO [RS:0;40249addbad9:43941 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:13,347 INFO [RS:0;40249addbad9:43941 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:13,347 INFO [RS:0;40249addbad9:43941 {}] hbase.ChoreService(168): Chore ScheduledChore name=40249addbad9,43941,1733279590307-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T02:33:13,363 INFO [RS:0;40249addbad9:43941 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T02:33:13,365 INFO [RS:0;40249addbad9:43941 {}] hbase.ChoreService(168): Chore ScheduledChore name=40249addbad9,43941,1733279590307-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:13,365 INFO [RS:0;40249addbad9:43941 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:13,365 INFO [RS:0;40249addbad9:43941 {}] regionserver.Replication(171): 40249addbad9,43941,1733279590307 started 2024-12-04T02:33:13,380 INFO [RS:0;40249addbad9:43941 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:13,381 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1482): Serving as 40249addbad9,43941,1733279590307, RpcServer on 40249addbad9/172.17.0.3:43941, sessionid=0x1019e2f9c250001 2024-12-04T02:33:13,381 DEBUG [RS:0;40249addbad9:43941 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T02:33:13,381 DEBUG [RS:0;40249addbad9:43941 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 40249addbad9,43941,1733279590307 2024-12-04T02:33:13,382 DEBUG [RS:0;40249addbad9:43941 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '40249addbad9,43941,1733279590307' 2024-12-04T02:33:13,382 DEBUG [RS:0;40249addbad9:43941 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T02:33:13,383 DEBUG [RS:0;40249addbad9:43941 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T02:33:13,383 DEBUG [RS:0;40249addbad9:43941 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T02:33:13,383 DEBUG [RS:0;40249addbad9:43941 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T02:33:13,384 DEBUG [RS:0;40249addbad9:43941 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 40249addbad9,43941,1733279590307 2024-12-04T02:33:13,384 DEBUG [RS:0;40249addbad9:43941 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '40249addbad9,43941,1733279590307' 2024-12-04T02:33:13,384 DEBUG [RS:0;40249addbad9:43941 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T02:33:13,385 DEBUG [RS:0;40249addbad9:43941 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T02:33:13,385 DEBUG [RS:0;40249addbad9:43941 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T02:33:13,385 INFO [RS:0;40249addbad9:43941 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T02:33:13,385 INFO [RS:0;40249addbad9:43941 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T02:33:13,594 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T02:33:13,595 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7 2024-12-04T02:33:13,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741832_1008 (size=32) 2024-12-04T02:33:14,010 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:33:14,013 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T02:33:14,016 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T02:33:14,017 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:33:14,018 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T02:33:14,019 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T02:33:14,022 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T02:33:14,022 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:33:14,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T02:33:14,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T02:33:14,026 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T02:33:14,027 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:33:14,028 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T02:33:14,028 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T02:33:14,031 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T02:33:14,032 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:33:14,033 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T02:33:14,033 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T02:33:14,035 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740 2024-12-04T02:33:14,036 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740 2024-12-04T02:33:14,040 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T02:33:14,040 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T02:33:14,042 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T02:33:14,048 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T02:33:14,058 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T02:33:14,059 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64298676, jitterRate=-0.04187506437301636}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T02:33:14,062 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733279594010Initializing all the Stores at 1733279594012 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733279594012Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733279594012Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733279594012Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733279594013 (+1 ms)Cleaning up temporary data from old regions at 1733279594040 (+27 ms)Region opened successfully at 1733279594062 (+22 ms) 2024-12-04T02:33:14,063 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T02:33:14,063 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T02:33:14,063 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T02:33:14,063 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T02:33:14,064 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T02:33:14,065 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T02:33:14,065 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733279594063Disabling compacts and flushes for region at 1733279594063Disabling writes for close at 1733279594063Writing region close event to WAL at 1733279594064 (+1 ms)Closed at 1733279594065 (+1 ms) 2024-12-04T02:33:14,073 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T02:33:14,073 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T02:33:14,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T02:33:14,095 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T02:33:14,100 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T02:33:14,252 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:15,253 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:16,253 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:17,254 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:18,255 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:19,255 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:19,900 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T02:33:19,901 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-04T02:33:20,259 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:21,259 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:22,260 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:23,261 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:24,261 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:25,262 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:26,263 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:27,264 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:28,264 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:29,265 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:30,266 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:31,267 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:32,267 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:33,268 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:34,269 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:35,270 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:36,270 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:37,271 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:38,272 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:38,390 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T02:33:39,273 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:40,274 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:41,274 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:42,275 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:43,276 WARN [40249addbad9:43601 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T02:33:43,390 INFO [RS:0;40249addbad9:43941 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T02:33:43,395 INFO [RS:0;40249addbad9:43941 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=40249addbad9%2C43941%2C1733279590307, suffix=, logDir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/WALs/40249addbad9,43941,1733279590307, archiveDir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/oldWALs, maxLogs=32 2024-12-04T02:33:43,414 DEBUG [RS:0;40249addbad9:43941 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/WALs/40249addbad9,43941,1733279590307/40249addbad9%2C43941%2C1733279590307.1733279623398, exclude list is [], retry=0 2024-12-04T02:33:43,418 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33875,DS-3967ebea-8971-42e1-aea7-2a8d7c5a9800,DISK] 2024-12-04T02:33:43,422 INFO [RS:0;40249addbad9:43941 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/WALs/40249addbad9,43941,1733279590307/40249addbad9%2C43941%2C1733279590307.1733279623398 2024-12-04T02:33:43,422 DEBUG [RS:0;40249addbad9:43941 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44099:44099)] 2024-12-04T02:33:43,528 DEBUG [40249addbad9:43601 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-04T02:33:43,540 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:33:43,546 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 40249addbad9,43941,1733279590307, state=OPENING 2024-12-04T02:33:43,606 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T02:33:43,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43941-0x1019e2f9c250001, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T02:33:43,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T02:33:43,618 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T02:33:43,618 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T02:33:43,620 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T02:33:43,622 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=40249addbad9,43941,1733279590307}] 2024-12-04T02:33:43,800 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T02:33:43,804 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57795, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T02:33:43,821 INFO [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T02:33:43,822 INFO [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T02:33:43,823 INFO [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-04T02:33:43,827 INFO [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=40249addbad9%2C43941%2C1733279590307.meta, suffix=.meta, logDir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/WALs/40249addbad9,43941,1733279590307, archiveDir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/oldWALs, maxLogs=32 2024-12-04T02:33:43,849 DEBUG [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/WALs/40249addbad9,43941,1733279590307/40249addbad9%2C43941%2C1733279590307.meta.1733279623829.meta, exclude list is [], retry=0 2024-12-04T02:33:43,853 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33875,DS-3967ebea-8971-42e1-aea7-2a8d7c5a9800,DISK] 2024-12-04T02:33:43,857 INFO [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/WALs/40249addbad9,43941,1733279590307/40249addbad9%2C43941%2C1733279590307.meta.1733279623829.meta 2024-12-04T02:33:43,858 DEBUG [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44099:44099)] 2024-12-04T02:33:43,858 DEBUG [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T02:33:43,860 DEBUG [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T02:33:43,864 DEBUG [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T02:33:43,870 INFO [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T02:33:43,876 DEBUG [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T02:33:43,877 DEBUG [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:33:43,878 DEBUG [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T02:33:43,878 DEBUG [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T02:33:43,882 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T02:33:43,884 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T02:33:43,884 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:33:43,885 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T02:33:43,885 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T02:33:43,887 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T02:33:43,887 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:33:43,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T02:33:43,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T02:33:43,890 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T02:33:43,890 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:33:43,891 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T02:33:43,892 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T02:33:43,893 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T02:33:43,894 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:33:43,895 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T02:33:43,895 DEBUG [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T02:33:43,898 DEBUG [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740 2024-12-04T02:33:43,902 DEBUG [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740 2024-12-04T02:33:43,909 DEBUG [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T02:33:43,909 DEBUG [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T02:33:43,911 DEBUG [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T02:33:43,914 DEBUG [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T02:33:43,916 INFO [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63149777, jitterRate=-0.058994993567466736}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T02:33:43,917 DEBUG [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T02:33:43,918 DEBUG [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733279623879Writing region info on filesystem at 1733279623879Initializing all the Stores at 1733279623881 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733279623881Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733279623882 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733279623882Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733279623882Cleaning up temporary data from old regions at 1733279623909 (+27 ms)Running coprocessor post-open hooks at 1733279623917 (+8 ms)Region opened successfully at 1733279623918 (+1 ms) 2024-12-04T02:33:43,926 INFO [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733279623790 2024-12-04T02:33:43,939 DEBUG [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T02:33:43,940 INFO [RS_OPEN_META-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T02:33:43,942 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:33:43,946 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 40249addbad9,43941,1733279590307, state=OPEN 2024-12-04T02:33:44,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43941-0x1019e2f9c250001, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T02:33:44,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T02:33:44,049 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T02:33:44,049 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T02:33:44,049 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=40249addbad9,43941,1733279590307 2024-12-04T02:33:44,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T02:33:44,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=40249addbad9,43941,1733279590307 in 427 msec 2024-12-04T02:33:44,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T02:33:44,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 29.9780 sec 2024-12-04T02:33:44,069 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T02:33:44,069 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T02:33:44,095 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T02:33:44,097 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=40249addbad9,43941,1733279590307, seqNum=-1] 2024-12-04T02:33:44,119 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T02:33:44,122 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42045, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T02:33:44,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 31.0590 sec 2024-12-04T02:33:44,141 INFO [master/40249addbad9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733279624141, completionTime=-1 2024-12-04T02:33:44,144 INFO [master/40249addbad9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-04T02:33:44,144 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T02:33:44,173 INFO [master/40249addbad9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-04T02:33:44,173 INFO [master/40249addbad9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733279684173 2024-12-04T02:33:44,173 INFO [master/40249addbad9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733279744173 2024-12-04T02:33:44,173 INFO [master/40249addbad9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-12-04T02:33:44,176 INFO [master/40249addbad9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=40249addbad9,43601,1733279589466-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:44,176 INFO [master/40249addbad9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=40249addbad9,43601,1733279589466-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:44,176 INFO [master/40249addbad9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=40249addbad9,43601,1733279589466-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:44,178 INFO [master/40249addbad9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-40249addbad9:43601, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:44,178 INFO [master/40249addbad9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:44,179 INFO [master/40249addbad9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:44,184 DEBUG [master/40249addbad9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T02:33:44,205 INFO [master/40249addbad9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.669sec 2024-12-04T02:33:44,206 INFO [master/40249addbad9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T02:33:44,208 INFO [master/40249addbad9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T02:33:44,208 INFO [master/40249addbad9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T02:33:44,209 INFO [master/40249addbad9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T02:33:44,209 INFO [master/40249addbad9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T02:33:44,210 INFO [master/40249addbad9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=40249addbad9,43601,1733279589466-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T02:33:44,210 INFO [master/40249addbad9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=40249addbad9,43601,1733279589466-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T02:33:44,219 DEBUG [master/40249addbad9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T02:33:44,219 INFO [master/40249addbad9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T02:33:44,220 INFO [master/40249addbad9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=40249addbad9,43601,1733279589466-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T02:33:44,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6915cddf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T02:33:44,306 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-04T02:33:44,306 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-04T02:33:44,308 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 40249addbad9,43601,-1 for getting cluster id 2024-12-04T02:33:44,310 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T02:33:44,317 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38a6c2e2-1cbb-4126-b063-575290e89383' 2024-12-04T02:33:44,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T02:33:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38a6c2e2-1cbb-4126-b063-575290e89383" 2024-12-04T02:33:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71fe4d5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T02:33:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [40249addbad9,43601,-1] 2024-12-04T02:33:44,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T02:33:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T02:33:44,331 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T02:33:44,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48d79b77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T02:33:44,335 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T02:33:44,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=40249addbad9,43941,1733279590307, seqNum=-1] 2024-12-04T02:33:44,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T02:33:44,344 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T02:33:44,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=40249addbad9,43601,1733279589466 2024-12-04T02:33:44,382 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a8b5254, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T02:33:44,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 40249addbad9,43601,-1 for getting cluster id 2024-12-04T02:33:44,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T02:33:44,385 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '38a6c2e2-1cbb-4126-b063-575290e89383' 2024-12-04T02:33:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T02:33:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "38a6c2e2-1cbb-4126-b063-575290e89383" 2024-12-04T02:33:44,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ab7d045, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T02:33:44,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [40249addbad9,43601,-1] 2024-12-04T02:33:44,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T02:33:44,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T02:33:44,393 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T02:33:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@666749f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T02:33:44,410 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=213, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=440, ProcessCount=11, AvailableMemoryMB=5058 2024-12-04T02:33:44,415 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T02:33:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 40249addbad9,43601,1733279589466 2024-12-04T02:33:44,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2fad16ae 2024-12-04T02:33:44,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T02:33:44,429 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T02:33:44,435 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T02:33:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-12-04T02:33:44,448 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T02:33:44,451 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:33:44,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-12-04T02:33:44,453 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T02:33:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T02:33:44,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741835_1011 (size=428) 2024-12-04T02:33:44,868 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 358638b58760ba89e229465286e54e93, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7 2024-12-04T02:33:44,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741836_1012 (size=79) 2024-12-04T02:33:45,285 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:33:45,285 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 358638b58760ba89e229465286e54e93, disabling compactions & flushes 2024-12-04T02:33:45,285 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93. 2024-12-04T02:33:45,285 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93. 2024-12-04T02:33:45,285 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93. after waiting 0 ms 2024-12-04T02:33:45,285 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93. 2024-12-04T02:33:45,285 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93. 2024-12-04T02:33:45,286 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 358638b58760ba89e229465286e54e93: Waiting for close lock at 1733279625285Disabling compacts and flushes for region at 1733279625285Disabling writes for close at 1733279625285Writing region close event to WAL at 1733279625285Closed at 1733279625285 2024-12-04T02:33:45,289 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T02:33:45,297 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733279625289"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733279625289"}]},"ts":"1733279625289"} 2024-12-04T02:33:45,301 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T02:33:45,303 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T02:33:45,305 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733279625303"}]},"ts":"1733279625303"} 2024-12-04T02:33:45,309 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-12-04T02:33:45,311 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=358638b58760ba89e229465286e54e93, ASSIGN}] 2024-12-04T02:33:45,314 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=358638b58760ba89e229465286e54e93, ASSIGN 2024-12-04T02:33:45,316 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=358638b58760ba89e229465286e54e93, ASSIGN; state=OFFLINE, location=40249addbad9,43941,1733279590307; forceNewPlan=false, retain=false 2024-12-04T02:33:45,468 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=358638b58760ba89e229465286e54e93, regionState=OPENING, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:33:45,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T02:33:45,478 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=358638b58760ba89e229465286e54e93, ASSIGN because future has completed 2024-12-04T02:33:45,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 358638b58760ba89e229465286e54e93, server=40249addbad9,43941,1733279590307}] 2024-12-04T02:33:45,641 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93. 2024-12-04T02:33:45,642 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 358638b58760ba89e229465286e54e93, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93.', STARTKEY => '', ENDKEY => ''} 2024-12-04T02:33:45,642 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 358638b58760ba89e229465286e54e93 2024-12-04T02:33:45,642 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:33:45,643 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 358638b58760ba89e229465286e54e93 2024-12-04T02:33:45,643 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 358638b58760ba89e229465286e54e93 2024-12-04T02:33:45,647 INFO [StoreOpener-358638b58760ba89e229465286e54e93-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 358638b58760ba89e229465286e54e93 2024-12-04T02:33:45,649 INFO [StoreOpener-358638b58760ba89e229465286e54e93-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 358638b58760ba89e229465286e54e93 columnFamilyName testFamily 2024-12-04T02:33:45,650 DEBUG [StoreOpener-358638b58760ba89e229465286e54e93-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:33:45,651 INFO [StoreOpener-358638b58760ba89e229465286e54e93-1 {}] regionserver.HStore(327): Store=358638b58760ba89e229465286e54e93/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T02:33:45,651 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 358638b58760ba89e229465286e54e93 2024-12-04T02:33:45,653 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testRetryOutOfOrderScannerNextExceptionAsync/358638b58760ba89e229465286e54e93 2024-12-04T02:33:45,654 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testRetryOutOfOrderScannerNextExceptionAsync/358638b58760ba89e229465286e54e93 2024-12-04T02:33:45,654 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 358638b58760ba89e229465286e54e93 2024-12-04T02:33:45,655 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 358638b58760ba89e229465286e54e93 2024-12-04T02:33:45,658 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 358638b58760ba89e229465286e54e93 2024-12-04T02:33:45,662 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testRetryOutOfOrderScannerNextExceptionAsync/358638b58760ba89e229465286e54e93/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T02:33:45,663 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 358638b58760ba89e229465286e54e93; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63420550, jitterRate=-0.054960161447525024}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T02:33:45,663 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 358638b58760ba89e229465286e54e93 2024-12-04T02:33:45,665 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 358638b58760ba89e229465286e54e93: Running coprocessor pre-open hook at 1733279625643Writing region info on filesystem at 1733279625643Initializing all the Stores at 1733279625645 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733279625645Cleaning up temporary data from old regions at 1733279625655 (+10 ms)Running coprocessor post-open hooks at 1733279625663 (+8 ms)Region opened successfully at 1733279625665 (+2 ms) 2024-12-04T02:33:45,667 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93., pid=6, masterSystemTime=1733279625635 2024-12-04T02:33:45,671 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93. 2024-12-04T02:33:45,672 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93. 2024-12-04T02:33:45,672 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=358638b58760ba89e229465286e54e93, regionState=OPEN, openSeqNum=2, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:33:45,677 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 358638b58760ba89e229465286e54e93, server=40249addbad9,43941,1733279590307 because future has completed 2024-12-04T02:33:45,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T02:33:45,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 358638b58760ba89e229465286e54e93, server=40249addbad9,43941,1733279590307 in 200 msec 2024-12-04T02:33:45,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T02:33:45,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=358638b58760ba89e229465286e54e93, ASSIGN in 373 msec 2024-12-04T02:33:45,692 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T02:33:45,693 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733279625692"}]},"ts":"1733279625692"} 2024-12-04T02:33:45,697 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-12-04T02:33:45,699 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T02:33:45,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2600 sec 2024-12-04T02:33:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T02:33:47,481 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-12-04T02:33:47,481 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-12-04T02:33:47,483 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T02:33:47,489 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-12-04T02:33:47,490 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T02:33:47,491 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-12-04T02:33:47,494 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T02:33:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=40249addbad9,43941,1733279590307, seqNum=-1] 2024-12-04T02:33:47,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T02:33:47,499 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T02:33:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93., hostname=40249addbad9,43941,1733279590307, seqNum=2] 2024-12-04T02:33:47,518 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-04T02:33:47,521 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-12-04T02:33:47,525 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-04T02:33:47,529 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-04T02:33:47,531 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-04T02:33:47,532 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=3447756289302593544 2024-12-04T02:33:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:39822 deadline: 1733279630531, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-04T02:33:47,555 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-04T02:33:47,557 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-04T02:33:47,557 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-04T02:33:47,559 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-04T02:33:47,560 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=3447756289302593546 2024-12-04T02:33:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:39822 deadline: 1733279630559, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-04T02:33:47,564 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-04T02:33:47,565 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=3447756289302593547 2024-12-04T02:33:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:39822 deadline: 1733279630563, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-04T02:33:47,568 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-04T02:33:47,569 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=3447756289302593548 2024-12-04T02:33:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:39822 deadline: 1733279630567, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-04T02:33:47,572 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-04T02:33:47,580 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=219 (was 213) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/40249addbad9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2055359103_22 at /127.0.0.1:51428 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=420 (was 440), ProcessCount=11 (was 11), AvailableMemoryMB=4886 (was 5058) 2024-12-04T02:33:47,588 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=219, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=420, ProcessCount=11, AvailableMemoryMB=4885 2024-12-04T02:33:47,591 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43601 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T02:33:47,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43601 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-12-04T02:33:47,596 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T02:33:47,597 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:33:47,597 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43601 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-12-04T02:33:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T02:33:47,599 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T02:33:47,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741837_1013 (size=414) 2024-12-04T02:33:48,011 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 46322ad3899db909759a023e550f5dc8, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7 2024-12-04T02:33:48,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741838_1014 (size=65) 2024-12-04T02:33:48,423 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:33:48,424 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 46322ad3899db909759a023e550f5dc8, disabling compactions & flushes 2024-12-04T02:33:48,424 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8. 2024-12-04T02:33:48,424 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8. 2024-12-04T02:33:48,424 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8. after waiting 0 ms 2024-12-04T02:33:48,424 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8. 2024-12-04T02:33:48,424 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8. 2024-12-04T02:33:48,424 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 46322ad3899db909759a023e550f5dc8: Waiting for close lock at 1733279628424Disabling compacts and flushes for region at 1733279628424Disabling writes for close at 1733279628424Writing region close event to WAL at 1733279628424Closed at 1733279628424 2024-12-04T02:33:48,426 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T02:33:48,427 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1733279628426"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733279628426"}]},"ts":"1733279628426"} 2024-12-04T02:33:48,430 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T02:33:48,433 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T02:33:48,433 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733279628433"}]},"ts":"1733279628433"} 2024-12-04T02:33:48,438 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-04T02:33:48,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=46322ad3899db909759a023e550f5dc8, ASSIGN}] 2024-12-04T02:33:48,440 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=46322ad3899db909759a023e550f5dc8, ASSIGN 2024-12-04T02:33:48,442 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=46322ad3899db909759a023e550f5dc8, ASSIGN; state=OFFLINE, location=40249addbad9,43941,1733279590307; forceNewPlan=false, retain=false 2024-12-04T02:33:48,593 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=46322ad3899db909759a023e550f5dc8, regionState=OPENING, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:33:48,597 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=46322ad3899db909759a023e550f5dc8, ASSIGN because future has completed 2024-12-04T02:33:48,598 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46322ad3899db909759a023e550f5dc8, server=40249addbad9,43941,1733279590307}] 2024-12-04T02:33:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T02:33:48,758 INFO [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8. 2024-12-04T02:33:48,758 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 46322ad3899db909759a023e550f5dc8, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8.', STARTKEY => '', ENDKEY => ''} 2024-12-04T02:33:48,759 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 46322ad3899db909759a023e550f5dc8 2024-12-04T02:33:48,759 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:33:48,759 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 46322ad3899db909759a023e550f5dc8 2024-12-04T02:33:48,759 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 46322ad3899db909759a023e550f5dc8 2024-12-04T02:33:48,761 INFO [StoreOpener-46322ad3899db909759a023e550f5dc8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 46322ad3899db909759a023e550f5dc8 2024-12-04T02:33:48,763 INFO [StoreOpener-46322ad3899db909759a023e550f5dc8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 46322ad3899db909759a023e550f5dc8 columnFamilyName testFamily 2024-12-04T02:33:48,763 DEBUG [StoreOpener-46322ad3899db909759a023e550f5dc8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:33:48,764 INFO [StoreOpener-46322ad3899db909759a023e550f5dc8-1 {}] regionserver.HStore(327): Store=46322ad3899db909759a023e550f5dc8/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T02:33:48,764 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 46322ad3899db909759a023e550f5dc8 2024-12-04T02:33:48,765 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnOpenScanner/46322ad3899db909759a023e550f5dc8 2024-12-04T02:33:48,766 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnOpenScanner/46322ad3899db909759a023e550f5dc8 2024-12-04T02:33:48,767 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 46322ad3899db909759a023e550f5dc8 2024-12-04T02:33:48,767 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 46322ad3899db909759a023e550f5dc8 2024-12-04T02:33:48,770 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 46322ad3899db909759a023e550f5dc8 2024-12-04T02:33:48,774 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnOpenScanner/46322ad3899db909759a023e550f5dc8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T02:33:48,775 INFO [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 46322ad3899db909759a023e550f5dc8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74444751, jitterRate=0.1093132346868515}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T02:33:48,775 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 46322ad3899db909759a023e550f5dc8 2024-12-04T02:33:48,776 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 46322ad3899db909759a023e550f5dc8: Running coprocessor pre-open hook at 1733279628759Writing region info on filesystem at 1733279628759Initializing all the Stores at 1733279628760 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733279628760Cleaning up temporary data from old regions at 1733279628767 (+7 ms)Running coprocessor post-open hooks at 1733279628776 (+9 ms)Region opened successfully at 1733279628776 2024-12-04T02:33:48,778 INFO [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8., pid=9, masterSystemTime=1733279628752 2024-12-04T02:33:48,782 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8. 2024-12-04T02:33:48,782 INFO [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8. 2024-12-04T02:33:48,783 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=46322ad3899db909759a023e550f5dc8, regionState=OPEN, openSeqNum=2, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:33:48,787 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 46322ad3899db909759a023e550f5dc8, server=40249addbad9,43941,1733279590307 because future has completed 2024-12-04T02:33:48,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-04T02:33:48,793 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 46322ad3899db909759a023e550f5dc8, server=40249addbad9,43941,1733279590307 in 191 msec 2024-12-04T02:33:48,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-04T02:33:48,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=46322ad3899db909759a023e550f5dc8, ASSIGN in 355 msec 2024-12-04T02:33:48,800 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T02:33:48,800 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733279628800"}]},"ts":"1733279628800"} 2024-12-04T02:33:48,803 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-04T02:33:48,805 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T02:33:48,808 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2130 sec 2024-12-04T02:33:49,414 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T02:33:49,439 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-12-04T02:33:49,442 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T02:33:49,443 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-12-04T02:33:49,900 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T02:33:49,900 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T02:33:49,901 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T02:33:49,901 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T02:33:49,901 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-04T02:33:49,901 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-12-04T02:33:49,902 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-04T02:33:49,902 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-04T02:33:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-04T02:33:50,649 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-12-04T02:33:50,649 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-04T02:33:50,649 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T02:33:50,654 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-04T02:33:50,654 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T02:33:50,655 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-12-04T02:33:50,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8., hostname=40249addbad9,43941,1733279590307, seqNum=2] 2024-12-04T02:33:50,671 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-04T02:33:50,671 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-04T02:33:50,675 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-04T02:33:50,677 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43941 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-04T02:33:56,687 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8., hostname=40249addbad9,43941,1733279590307, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8., hostname=40249addbad9,43941,1733279590307, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=40249addbad9:43941 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6008ms, rpcTimeout=5999ms 2024-12-04T02:33:56,688 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8., hostname=40249addbad9,43941,1733279590307, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6008ms, rpcTimeout=5999ms 2024-12-04T02:33:56,688 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8., hostname=40249addbad9,43941,1733279590307, seqNum=2 from cache 2024-12-04T02:33:56,699 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=222 (was 219) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=378 (was 420), ProcessCount=11 (was 11), AvailableMemoryMB=4710 (was 4885) 2024-12-04T02:33:56,708 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=378, ProcessCount=11, AvailableMemoryMB=4709 2024-12-04T02:33:56,711 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43601 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T02:33:56,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43601 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-12-04T02:33:56,715 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T02:33:56,715 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:33:56,715 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43601 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-12-04T02:33:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-04T02:33:56,718 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T02:33:56,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741839_1015 (size=412) 2024-12-04T02:33:57,139 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => be4fba22bce0f0993f9e6bdd23cbac06, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7 2024-12-04T02:33:57,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741840_1016 (size=63) 2024-12-04T02:33:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-12-04T02:33:57,557 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:33:57,557 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing be4fba22bce0f0993f9e6bdd23cbac06, disabling compactions & flushes 2024-12-04T02:33:57,557 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06. 2024-12-04T02:33:57,557 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06. 2024-12-04T02:33:57,557 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06. after waiting 0 ms 2024-12-04T02:33:57,557 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06. 2024-12-04T02:33:57,558 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06. 2024-12-04T02:33:57,558 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for be4fba22bce0f0993f9e6bdd23cbac06: Waiting for close lock at 1733279637557Disabling compacts and flushes for region at 1733279637557Disabling writes for close at 1733279637557Writing region close event to WAL at 1733279637557Closed at 1733279637557 2024-12-04T02:33:57,560 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T02:33:57,561 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1733279637560"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733279637560"}]},"ts":"1733279637560"} 2024-12-04T02:33:57,565 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T02:33:57,568 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T02:33:57,568 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733279637568"}]},"ts":"1733279637568"} 2024-12-04T02:33:57,571 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-04T02:33:57,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=be4fba22bce0f0993f9e6bdd23cbac06, ASSIGN}] 2024-12-04T02:33:57,574 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=be4fba22bce0f0993f9e6bdd23cbac06, ASSIGN 2024-12-04T02:33:57,576 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=be4fba22bce0f0993f9e6bdd23cbac06, ASSIGN; state=OFFLINE, location=40249addbad9,43941,1733279590307; forceNewPlan=false, retain=false 2024-12-04T02:33:57,727 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=be4fba22bce0f0993f9e6bdd23cbac06, regionState=OPENING, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:33:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-04T02:33:57,734 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=be4fba22bce0f0993f9e6bdd23cbac06, ASSIGN because future has completed 2024-12-04T02:33:57,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure be4fba22bce0f0993f9e6bdd23cbac06, server=40249addbad9,43941,1733279590307}] 2024-12-04T02:33:57,901 INFO [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06. 2024-12-04T02:33:57,902 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => be4fba22bce0f0993f9e6bdd23cbac06, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06.', STARTKEY => '', ENDKEY => ''} 2024-12-04T02:33:57,902 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:33:57,902 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:33:57,903 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:33:57,903 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:33:57,906 INFO [StoreOpener-be4fba22bce0f0993f9e6bdd23cbac06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:33:57,908 INFO [StoreOpener-be4fba22bce0f0993f9e6bdd23cbac06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be4fba22bce0f0993f9e6bdd23cbac06 columnFamilyName testFamily 2024-12-04T02:33:57,908 DEBUG [StoreOpener-be4fba22bce0f0993f9e6bdd23cbac06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:33:57,909 INFO [StoreOpener-be4fba22bce0f0993f9e6bdd23cbac06-1 {}] regionserver.HStore(327): Store=be4fba22bce0f0993f9e6bdd23cbac06/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T02:33:57,909 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:33:57,911 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnNextAsync/be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:33:57,912 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnNextAsync/be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:33:57,912 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:33:57,912 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:33:57,916 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:33:57,920 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnNextAsync/be4fba22bce0f0993f9e6bdd23cbac06/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T02:33:57,921 INFO [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened be4fba22bce0f0993f9e6bdd23cbac06; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65314210, jitterRate=-0.02674242854118347}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T02:33:57,921 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:33:57,922 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for be4fba22bce0f0993f9e6bdd23cbac06: Running coprocessor pre-open hook at 1733279637903Writing region info on filesystem at 1733279637903Initializing all the Stores at 1733279637905 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733279637905Cleaning up temporary data from old regions at 1733279637912 (+7 ms)Running coprocessor post-open hooks at 1733279637921 (+9 ms)Region opened successfully at 1733279637922 (+1 ms) 2024-12-04T02:33:57,923 INFO [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06., pid=12, masterSystemTime=1733279637890 2024-12-04T02:33:57,927 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06. 2024-12-04T02:33:57,927 INFO [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06. 2024-12-04T02:33:57,928 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=be4fba22bce0f0993f9e6bdd23cbac06, regionState=OPEN, openSeqNum=2, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:33:57,932 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure be4fba22bce0f0993f9e6bdd23cbac06, server=40249addbad9,43941,1733279590307 because future has completed 2024-12-04T02:33:57,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-04T02:33:57,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure be4fba22bce0f0993f9e6bdd23cbac06, server=40249addbad9,43941,1733279590307 in 199 msec 2024-12-04T02:33:57,945 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-04T02:33:57,946 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=be4fba22bce0f0993f9e6bdd23cbac06, ASSIGN in 368 msec 2024-12-04T02:33:57,947 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T02:33:57,948 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733279637948"}]},"ts":"1733279637948"} 2024-12-04T02:33:57,951 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-04T02:33:57,953 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T02:33:57,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2420 sec 2024-12-04T02:33:59,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-04T02:33:59,750 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-12-04T02:33:59,750 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-04T02:33:59,750 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T02:33:59,755 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-04T02:33:59,756 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T02:33:59,756 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-12-04T02:33:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06., hostname=40249addbad9,43941,1733279590307, seqNum=2] 2024-12-04T02:33:59,777 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-04T02:33:59,777 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-04T02:33:59,783 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-04T02:33:59,788 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-04T02:33:59,791 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-04T02:33:59,791 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43941 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-04T02:33:59,900 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-04T02:33:59,900 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-04T02:34:01,445 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-12-04T02:34:03,438 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T02:34:05,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43941 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '3447756289302593557' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-04T02:34:05,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43941 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:39822 deadline: 1733279651804, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '3447756289302593557' does not exist 2024-12-04T02:34:05,812 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 40249addbad9,43941,1733279590307 for closing scanner id = 3447756289302593557 for be4fba22bce0f0993f9e6bdd23cbac06 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '3447756289302593557' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T02:34:05,814 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=224 (was 222) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=342 (was 378), ProcessCount=11 (was 11), AvailableMemoryMB=4664 (was 4709) 2024-12-04T02:34:05,824 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=224, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=342, ProcessCount=11, AvailableMemoryMB=4663 2024-12-04T02:34:05,827 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T02:34:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-12-04T02:34:05,832 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T02:34:05,832 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:34:05,832 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-12-04T02:34:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T02:34:05,834 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T02:34:05,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741841_1017 (size=423) 2024-12-04T02:34:06,247 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 20d9ebc06c69a540e5cbfa93071b4415, NAME => 'testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7 2024-12-04T02:34:06,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741842_1018 (size=74) 2024-12-04T02:34:06,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-12-04T02:34:06,657 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:34:06,657 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 20d9ebc06c69a540e5cbfa93071b4415, disabling compactions & flushes 2024-12-04T02:34:06,658 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415. 2024-12-04T02:34:06,658 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415. 2024-12-04T02:34:06,658 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415. after waiting 0 ms 2024-12-04T02:34:06,658 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415. 2024-12-04T02:34:06,658 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415. 2024-12-04T02:34:06,658 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 20d9ebc06c69a540e5cbfa93071b4415: Waiting for close lock at 1733279646657Disabling compacts and flushes for region at 1733279646657Disabling writes for close at 1733279646658 (+1 ms)Writing region close event to WAL at 1733279646658Closed at 1733279646658 2024-12-04T02:34:06,660 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T02:34:06,660 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733279646660"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733279646660"}]},"ts":"1733279646660"} 2024-12-04T02:34:06,663 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T02:34:06,664 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T02:34:06,665 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733279646664"}]},"ts":"1733279646664"} 2024-12-04T02:34:06,667 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-12-04T02:34:06,668 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=20d9ebc06c69a540e5cbfa93071b4415, ASSIGN}] 2024-12-04T02:34:06,669 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=20d9ebc06c69a540e5cbfa93071b4415, ASSIGN 2024-12-04T02:34:06,671 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=20d9ebc06c69a540e5cbfa93071b4415, ASSIGN; state=OFFLINE, location=40249addbad9,43941,1733279590307; forceNewPlan=false, retain=false 2024-12-04T02:34:06,821 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=20d9ebc06c69a540e5cbfa93071b4415, regionState=OPENING, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:34:06,825 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=20d9ebc06c69a540e5cbfa93071b4415, ASSIGN because future has completed 2024-12-04T02:34:06,825 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 20d9ebc06c69a540e5cbfa93071b4415, server=40249addbad9,43941,1733279590307}] 2024-12-04T02:34:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T02:34:06,983 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415. 2024-12-04T02:34:06,983 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 20d9ebc06c69a540e5cbfa93071b4415, NAME => 'testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415.', STARTKEY => '', ENDKEY => ''} 2024-12-04T02:34:06,983 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 20d9ebc06c69a540e5cbfa93071b4415 2024-12-04T02:34:06,984 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:34:06,984 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 20d9ebc06c69a540e5cbfa93071b4415 2024-12-04T02:34:06,984 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 20d9ebc06c69a540e5cbfa93071b4415 2024-12-04T02:34:06,986 INFO [StoreOpener-20d9ebc06c69a540e5cbfa93071b4415-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 20d9ebc06c69a540e5cbfa93071b4415 2024-12-04T02:34:06,989 INFO [StoreOpener-20d9ebc06c69a540e5cbfa93071b4415-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20d9ebc06c69a540e5cbfa93071b4415 columnFamilyName testFamily 2024-12-04T02:34:06,989 DEBUG [StoreOpener-20d9ebc06c69a540e5cbfa93071b4415-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:34:06,990 INFO [StoreOpener-20d9ebc06c69a540e5cbfa93071b4415-1 {}] regionserver.HStore(327): Store=20d9ebc06c69a540e5cbfa93071b4415/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T02:34:06,990 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 20d9ebc06c69a540e5cbfa93071b4415 2024-12-04T02:34:06,992 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testRetryOutOfOrderScannerNextException/20d9ebc06c69a540e5cbfa93071b4415 2024-12-04T02:34:06,993 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testRetryOutOfOrderScannerNextException/20d9ebc06c69a540e5cbfa93071b4415 2024-12-04T02:34:06,994 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 20d9ebc06c69a540e5cbfa93071b4415 2024-12-04T02:34:06,994 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 20d9ebc06c69a540e5cbfa93071b4415 2024-12-04T02:34:06,999 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 20d9ebc06c69a540e5cbfa93071b4415 2024-12-04T02:34:07,003 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testRetryOutOfOrderScannerNextException/20d9ebc06c69a540e5cbfa93071b4415/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T02:34:07,004 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 20d9ebc06c69a540e5cbfa93071b4415; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66784986, jitterRate=-0.004826158285140991}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T02:34:07,004 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 20d9ebc06c69a540e5cbfa93071b4415 2024-12-04T02:34:07,004 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 20d9ebc06c69a540e5cbfa93071b4415: Running coprocessor pre-open hook at 1733279646984Writing region info on filesystem at 1733279646984Initializing all the Stores at 1733279646986 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733279646986Cleaning up temporary data from old regions at 1733279646994 (+8 ms)Running coprocessor post-open hooks at 1733279647004 (+10 ms)Region opened successfully at 1733279647004 2024-12-04T02:34:07,006 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415., pid=15, masterSystemTime=1733279646978 2024-12-04T02:34:07,009 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415. 2024-12-04T02:34:07,009 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415. 2024-12-04T02:34:07,010 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=20d9ebc06c69a540e5cbfa93071b4415, regionState=OPEN, openSeqNum=2, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:34:07,013 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 20d9ebc06c69a540e5cbfa93071b4415, server=40249addbad9,43941,1733279590307 because future has completed 2024-12-04T02:34:07,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-04T02:34:07,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 20d9ebc06c69a540e5cbfa93071b4415, server=40249addbad9,43941,1733279590307 in 190 msec 2024-12-04T02:34:07,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-04T02:34:07,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=20d9ebc06c69a540e5cbfa93071b4415, ASSIGN in 352 msec 2024-12-04T02:34:07,026 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T02:34:07,026 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733279647026"}]},"ts":"1733279647026"} 2024-12-04T02:34:07,029 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-12-04T02:34:07,031 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T02:34:07,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2040 sec 2024-12-04T02:34:07,446 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-12-04T02:34:08,391 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T02:34:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-04T02:34:08,861 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-12-04T02:34:08,861 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-12-04T02:34:08,861 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T02:34:08,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-12-04T02:34:08,868 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T02:34:08,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-12-04T02:34:08,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415., hostname=40249addbad9,43941,1733279590307, seqNum=2] 2024-12-04T02:34:08,886 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-04T02:34:08,886 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-12-04T02:34:08,889 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-04T02:34:08,891 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-04T02:34:08,892 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-04T02:34:08,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=3447756289302593561 2024-12-04T02:34:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:39822 deadline: 1733279651892, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-04T02:34:08,896 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-04T02:34:08,897 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-04T02:34:08,898 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-04T02:34:08,899 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-04T02:34:08,900 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=3447756289302593563 2024-12-04T02:34:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:39822 deadline: 1733279651898, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-04T02:34:08,902 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-04T02:34:08,903 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=3447756289302593564 2024-12-04T02:34:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:39822 deadline: 1733279651902, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-04T02:34:08,906 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-04T02:34:08,907 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=3447756289302593565 2024-12-04T02:34:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:39822 deadline: 1733279651905, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-04T02:34:08,909 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-04T02:34:08,917 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 224), OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=342 (was 342), ProcessCount=11 (was 11), AvailableMemoryMB=4575 (was 4663) 2024-12-04T02:34:08,924 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=342, ProcessCount=11, AvailableMemoryMB=4574 2024-12-04T02:34:08,926 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T02:34:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-12-04T02:34:08,929 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T02:34:08,929 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:34:08,929 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-12-04T02:34:08,931 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T02:34:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-04T02:34:08,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741843_1019 (size=418) 2024-12-04T02:34:09,340 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a921ec0371ebd57e4fc59d2d75b9cee9, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7 2024-12-04T02:34:09,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741844_1020 (size=69) 2024-12-04T02:34:09,751 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:34:09,751 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing a921ec0371ebd57e4fc59d2d75b9cee9, disabling compactions & flushes 2024-12-04T02:34:09,751 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9. 2024-12-04T02:34:09,751 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9. 2024-12-04T02:34:09,752 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9. after waiting 0 ms 2024-12-04T02:34:09,752 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9. 2024-12-04T02:34:09,752 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9. 2024-12-04T02:34:09,752 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for a921ec0371ebd57e4fc59d2d75b9cee9: Waiting for close lock at 1733279649751Disabling compacts and flushes for region at 1733279649751Disabling writes for close at 1733279649752 (+1 ms)Writing region close event to WAL at 1733279649752Closed at 1733279649752 2024-12-04T02:34:09,754 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T02:34:09,755 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1733279649755"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733279649755"}]},"ts":"1733279649755"} 2024-12-04T02:34:09,765 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T02:34:09,768 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T02:34:09,768 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733279649768"}]},"ts":"1733279649768"} 2024-12-04T02:34:09,773 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-04T02:34:09,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=a921ec0371ebd57e4fc59d2d75b9cee9, ASSIGN}] 2024-12-04T02:34:09,777 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=a921ec0371ebd57e4fc59d2d75b9cee9, ASSIGN 2024-12-04T02:34:09,779 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=a921ec0371ebd57e4fc59d2d75b9cee9, ASSIGN; state=OFFLINE, location=40249addbad9,43941,1733279590307; forceNewPlan=false, retain=false 2024-12-04T02:34:09,900 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-04T02:34:09,900 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-12-04T02:34:09,930 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=a921ec0371ebd57e4fc59d2d75b9cee9, regionState=OPENING, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:34:09,934 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=a921ec0371ebd57e4fc59d2d75b9cee9, ASSIGN because future has completed 2024-12-04T02:34:09,935 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure a921ec0371ebd57e4fc59d2d75b9cee9, server=40249addbad9,43941,1733279590307}] 2024-12-04T02:34:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-04T02:34:10,094 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9. 2024-12-04T02:34:10,094 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => a921ec0371ebd57e4fc59d2d75b9cee9, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9.', STARTKEY => '', ENDKEY => ''} 2024-12-04T02:34:10,095 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner a921ec0371ebd57e4fc59d2d75b9cee9 2024-12-04T02:34:10,095 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:34:10,095 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for a921ec0371ebd57e4fc59d2d75b9cee9 2024-12-04T02:34:10,095 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for a921ec0371ebd57e4fc59d2d75b9cee9 2024-12-04T02:34:10,097 INFO [StoreOpener-a921ec0371ebd57e4fc59d2d75b9cee9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a921ec0371ebd57e4fc59d2d75b9cee9 2024-12-04T02:34:10,099 INFO [StoreOpener-a921ec0371ebd57e4fc59d2d75b9cee9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a921ec0371ebd57e4fc59d2d75b9cee9 columnFamilyName testFamily 2024-12-04T02:34:10,099 DEBUG [StoreOpener-a921ec0371ebd57e4fc59d2d75b9cee9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:34:10,100 INFO [StoreOpener-a921ec0371ebd57e4fc59d2d75b9cee9-1 {}] regionserver.HStore(327): Store=a921ec0371ebd57e4fc59d2d75b9cee9/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T02:34:10,100 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for a921ec0371ebd57e4fc59d2d75b9cee9 2024-12-04T02:34:10,102 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnOpenScanner/a921ec0371ebd57e4fc59d2d75b9cee9 2024-12-04T02:34:10,102 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnOpenScanner/a921ec0371ebd57e4fc59d2d75b9cee9 2024-12-04T02:34:10,103 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for a921ec0371ebd57e4fc59d2d75b9cee9 2024-12-04T02:34:10,103 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for a921ec0371ebd57e4fc59d2d75b9cee9 2024-12-04T02:34:10,105 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for a921ec0371ebd57e4fc59d2d75b9cee9 2024-12-04T02:34:10,108 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnOpenScanner/a921ec0371ebd57e4fc59d2d75b9cee9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T02:34:10,109 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened a921ec0371ebd57e4fc59d2d75b9cee9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59757600, jitterRate=-0.1095423698425293}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T02:34:10,109 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a921ec0371ebd57e4fc59d2d75b9cee9 2024-12-04T02:34:10,110 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for a921ec0371ebd57e4fc59d2d75b9cee9: Running coprocessor pre-open hook at 1733279650095Writing region info on filesystem at 1733279650095Initializing all the Stores at 1733279650096 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733279650097 (+1 ms)Cleaning up temporary data from old regions at 1733279650103 (+6 ms)Running coprocessor post-open hooks at 1733279650109 (+6 ms)Region opened successfully at 1733279650110 (+1 ms) 2024-12-04T02:34:10,111 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9., pid=18, masterSystemTime=1733279650088 2024-12-04T02:34:10,114 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9. 2024-12-04T02:34:10,114 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9. 2024-12-04T02:34:10,115 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=a921ec0371ebd57e4fc59d2d75b9cee9, regionState=OPEN, openSeqNum=2, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:34:10,118 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure a921ec0371ebd57e4fc59d2d75b9cee9, server=40249addbad9,43941,1733279590307 because future has completed 2024-12-04T02:34:10,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-04T02:34:10,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure a921ec0371ebd57e4fc59d2d75b9cee9, server=40249addbad9,43941,1733279590307 in 184 msec 2024-12-04T02:34:10,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-04T02:34:10,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=a921ec0371ebd57e4fc59d2d75b9cee9, ASSIGN in 350 msec 2024-12-04T02:34:10,128 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T02:34:10,129 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733279650128"}]},"ts":"1733279650128"} 2024-12-04T02:34:10,131 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-04T02:34:10,133 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T02:34:10,136 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2070 sec 2024-12-04T02:34:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-04T02:34:11,970 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-12-04T02:34:11,970 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-04T02:34:11,970 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T02:34:11,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-04T02:34:11,977 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T02:34:11,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-12-04T02:34:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9., hostname=40249addbad9,43941,1733279590307, seqNum=2] 2024-12-04T02:34:11,994 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-04T02:34:11,995 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-04T02:34:11,999 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-04T02:34:12,001 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-04T02:34:12,486 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T02:34:13,009 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9., hostname=40249addbad9,43941,1733279590307, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9., hostname=40249addbad9,43941,1733279590307, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=40249addbad9:43941 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-12-04T02:34:13,009 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9., hostname=40249addbad9,43941,1733279590307, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-12-04T02:34:13,009 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9., hostname=40249addbad9,43941,1733279590307, seqNum=2 from cache 2024-12-04T02:34:13,447 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-12-04T02:34:13,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-12-04T02:34:14,024 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9., hostname=40249addbad9,43941,1733279590307, seqNum=2] 2024-12-04T02:34:14,025 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-04T02:34:15,009 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9., hostname=40249addbad9,43941,1733279590307, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9., hostname=40249addbad9,43941,1733279590307, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=40249addbad9:43941 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=984ms, rpcTimeout=974ms 2024-12-04T02:34:15,009 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9., hostname=40249addbad9,43941,1733279590307, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=984ms, rpcTimeout=974ms 2024-12-04T02:34:15,009 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9., hostname=40249addbad9,43941,1733279590307, seqNum=2 from cache 2024-12-04T02:34:15,018 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 223), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=314 (was 342), ProcessCount=11 (was 11), AvailableMemoryMB=4394 (was 4574) 2024-12-04T02:34:15,026 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=314, ProcessCount=11, AvailableMemoryMB=4392 2024-12-04T02:34:15,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T02:34:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-12-04T02:34:15,031 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T02:34:15,032 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:34:15,032 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-12-04T02:34:15,033 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T02:34:15,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T02:34:15,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741845_1021 (size=416) 2024-12-04T02:34:15,042 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 53b175aca7affc4c1f7e036710ba4734, NAME => 'testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7 2024-12-04T02:34:15,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741846_1022 (size=67) 2024-12-04T02:34:15,453 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:34:15,453 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 53b175aca7affc4c1f7e036710ba4734, disabling compactions & flushes 2024-12-04T02:34:15,453 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734. 2024-12-04T02:34:15,454 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734. 2024-12-04T02:34:15,454 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734. after waiting 0 ms 2024-12-04T02:34:15,454 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734. 2024-12-04T02:34:15,454 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734. 2024-12-04T02:34:15,455 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 53b175aca7affc4c1f7e036710ba4734: Waiting for close lock at 1733279655453Disabling compacts and flushes for region at 1733279655453Disabling writes for close at 1733279655454 (+1 ms)Writing region close event to WAL at 1733279655454Closed at 1733279655454 2024-12-04T02:34:15,458 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T02:34:15,458 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1733279655458"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733279655458"}]},"ts":"1733279655458"} 2024-12-04T02:34:15,463 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T02:34:15,465 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T02:34:15,465 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733279655465"}]},"ts":"1733279655465"} 2024-12-04T02:34:15,469 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-04T02:34:15,470 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=53b175aca7affc4c1f7e036710ba4734, ASSIGN}] 2024-12-04T02:34:15,472 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=53b175aca7affc4c1f7e036710ba4734, ASSIGN 2024-12-04T02:34:15,474 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=53b175aca7affc4c1f7e036710ba4734, ASSIGN; state=OFFLINE, location=40249addbad9,43941,1733279590307; forceNewPlan=false, retain=false 2024-12-04T02:34:15,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-12-04T02:34:15,625 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=53b175aca7affc4c1f7e036710ba4734, regionState=OPENING, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:34:15,629 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=53b175aca7affc4c1f7e036710ba4734, ASSIGN because future has completed 2024-12-04T02:34:15,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53b175aca7affc4c1f7e036710ba4734, server=40249addbad9,43941,1733279590307}] 2024-12-04T02:34:15,793 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734. 2024-12-04T02:34:15,793 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 53b175aca7affc4c1f7e036710ba4734, NAME => 'testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734.', STARTKEY => '', ENDKEY => ''} 2024-12-04T02:34:15,794 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 53b175aca7affc4c1f7e036710ba4734 2024-12-04T02:34:15,794 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:34:15,794 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 53b175aca7affc4c1f7e036710ba4734 2024-12-04T02:34:15,794 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 53b175aca7affc4c1f7e036710ba4734 2024-12-04T02:34:15,797 INFO [StoreOpener-53b175aca7affc4c1f7e036710ba4734-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 53b175aca7affc4c1f7e036710ba4734 2024-12-04T02:34:15,799 INFO [StoreOpener-53b175aca7affc4c1f7e036710ba4734-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 53b175aca7affc4c1f7e036710ba4734 columnFamilyName testFamily 2024-12-04T02:34:15,800 DEBUG [StoreOpener-53b175aca7affc4c1f7e036710ba4734-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:34:15,800 INFO [StoreOpener-53b175aca7affc4c1f7e036710ba4734-1 {}] regionserver.HStore(327): Store=53b175aca7affc4c1f7e036710ba4734/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T02:34:15,801 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 53b175aca7affc4c1f7e036710ba4734 2024-12-04T02:34:15,802 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnNextAsync/53b175aca7affc4c1f7e036710ba4734 2024-12-04T02:34:15,803 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnNextAsync/53b175aca7affc4c1f7e036710ba4734 2024-12-04T02:34:15,804 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 53b175aca7affc4c1f7e036710ba4734 2024-12-04T02:34:15,804 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 53b175aca7affc4c1f7e036710ba4734 2024-12-04T02:34:15,807 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 53b175aca7affc4c1f7e036710ba4734 2024-12-04T02:34:15,811 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnNextAsync/53b175aca7affc4c1f7e036710ba4734/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T02:34:15,812 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 53b175aca7affc4c1f7e036710ba4734; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71922311, jitterRate=0.07172594964504242}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T02:34:15,812 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 53b175aca7affc4c1f7e036710ba4734 2024-12-04T02:34:15,813 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 53b175aca7affc4c1f7e036710ba4734: Running coprocessor pre-open hook at 1733279655795Writing region info on filesystem at 1733279655795Initializing all the Stores at 1733279655797 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733279655797Cleaning up temporary data from old regions at 1733279655804 (+7 ms)Running coprocessor post-open hooks at 1733279655812 (+8 ms)Region opened successfully at 1733279655813 (+1 ms) 2024-12-04T02:34:15,815 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734., pid=21, masterSystemTime=1733279655783 2024-12-04T02:34:15,818 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734. 2024-12-04T02:34:15,818 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734. 2024-12-04T02:34:15,818 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=53b175aca7affc4c1f7e036710ba4734, regionState=OPEN, openSeqNum=2, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:34:15,821 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53b175aca7affc4c1f7e036710ba4734, server=40249addbad9,43941,1733279590307 because future has completed 2024-12-04T02:34:15,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-04T02:34:15,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 53b175aca7affc4c1f7e036710ba4734, server=40249addbad9,43941,1733279590307 in 194 msec 2024-12-04T02:34:15,831 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-04T02:34:15,831 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=53b175aca7affc4c1f7e036710ba4734, ASSIGN in 356 msec 2024-12-04T02:34:15,832 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T02:34:15,832 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733279655832"}]},"ts":"1733279655832"} 2024-12-04T02:34:15,835 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-04T02:34:15,836 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T02:34:15,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 808 msec 2024-12-04T02:34:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-04T02:34:16,050 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-12-04T02:34:16,050 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-04T02:34:16,050 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T02:34:16,055 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-04T02:34:16,055 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T02:34:16,056 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-12-04T02:34:16,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734., hostname=40249addbad9,43941,1733279590307, seqNum=2] 2024-12-04T02:34:16,071 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-04T02:34:16,072 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-04T02:34:16,076 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-04T02:34:16,079 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-04T02:34:16,080 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-04T02:34:16,081 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-04T02:34:19,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43941 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '3447756289302593576' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-04T02:34:19,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43941 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:39822 deadline: 1733279660089, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '3447756289302593576' does not exist 2024-12-04T02:34:19,093 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 40249addbad9,43941,1733279590307 for closing scanner id = 3447756289302593576 for 53b175aca7affc4c1f7e036710ba4734 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '3447756289302593576' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T02:34:19,100 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=223 (was 222) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2055359103_22 at /127.0.0.1:47054 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=305 (was 314), ProcessCount=11 (was 11), AvailableMemoryMB=4319 (was 4392) 2024-12-04T02:34:19,108 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=305, ProcessCount=11, AvailableMemoryMB=4319 2024-12-04T02:34:19,110 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T02:34:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-12-04T02:34:19,113 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T02:34:19,113 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:34:19,113 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-12-04T02:34:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-04T02:34:19,115 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T02:34:19,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741847_1023 (size=411) 2024-12-04T02:34:19,449 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-12-04T02:34:19,526 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 05e988be3b28b57da7eaaf731591b3b5, NAME => 'testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7 2024-12-04T02:34:19,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741848_1024 (size=62) 2024-12-04T02:34:19,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-12-04T02:34:19,900 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-04T02:34:19,900 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-04T02:34:19,901 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-04T02:34:19,901 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-04T02:34:19,936 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:34:19,936 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 05e988be3b28b57da7eaaf731591b3b5, disabling compactions & flushes 2024-12-04T02:34:19,936 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5. 2024-12-04T02:34:19,936 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5. 2024-12-04T02:34:19,936 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5. after waiting 0 ms 2024-12-04T02:34:19,936 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5. 2024-12-04T02:34:19,937 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5. 2024-12-04T02:34:19,937 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 05e988be3b28b57da7eaaf731591b3b5: Waiting for close lock at 1733279659936Disabling compacts and flushes for region at 1733279659936Disabling writes for close at 1733279659936Writing region close event to WAL at 1733279659936Closed at 1733279659936 2024-12-04T02:34:19,940 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T02:34:19,941 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1733279659940"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733279659940"}]},"ts":"1733279659940"} 2024-12-04T02:34:19,946 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T02:34:19,949 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T02:34:19,949 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733279659949"}]},"ts":"1733279659949"} 2024-12-04T02:34:19,952 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-04T02:34:19,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=05e988be3b28b57da7eaaf731591b3b5, ASSIGN}] 2024-12-04T02:34:19,955 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=05e988be3b28b57da7eaaf731591b3b5, ASSIGN 2024-12-04T02:34:19,956 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=05e988be3b28b57da7eaaf731591b3b5, ASSIGN; state=OFFLINE, location=40249addbad9,43941,1733279590307; forceNewPlan=false, retain=false 2024-12-04T02:34:20,107 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=05e988be3b28b57da7eaaf731591b3b5, regionState=OPENING, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:34:20,112 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=05e988be3b28b57da7eaaf731591b3b5, ASSIGN because future has completed 2024-12-04T02:34:20,112 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05e988be3b28b57da7eaaf731591b3b5, server=40249addbad9,43941,1733279590307}] 2024-12-04T02:34:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-04T02:34:20,274 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5. 2024-12-04T02:34:20,275 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 05e988be3b28b57da7eaaf731591b3b5, NAME => 'testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5.', STARTKEY => '', ENDKEY => ''} 2024-12-04T02:34:20,275 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 05e988be3b28b57da7eaaf731591b3b5 2024-12-04T02:34:20,275 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:34:20,276 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 05e988be3b28b57da7eaaf731591b3b5 2024-12-04T02:34:20,276 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 05e988be3b28b57da7eaaf731591b3b5 2024-12-04T02:34:20,278 INFO [StoreOpener-05e988be3b28b57da7eaaf731591b3b5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 05e988be3b28b57da7eaaf731591b3b5 2024-12-04T02:34:20,280 INFO [StoreOpener-05e988be3b28b57da7eaaf731591b3b5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 05e988be3b28b57da7eaaf731591b3b5 columnFamilyName testFamily 2024-12-04T02:34:20,280 DEBUG [StoreOpener-05e988be3b28b57da7eaaf731591b3b5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:34:20,281 INFO [StoreOpener-05e988be3b28b57da7eaaf731591b3b5-1 {}] regionserver.HStore(327): Store=05e988be3b28b57da7eaaf731591b3b5/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T02:34:20,281 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 05e988be3b28b57da7eaaf731591b3b5 2024-12-04T02:34:20,283 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnNext/05e988be3b28b57da7eaaf731591b3b5 2024-12-04T02:34:20,284 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnNext/05e988be3b28b57da7eaaf731591b3b5 2024-12-04T02:34:20,284 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 05e988be3b28b57da7eaaf731591b3b5 2024-12-04T02:34:20,284 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 05e988be3b28b57da7eaaf731591b3b5 2024-12-04T02:34:20,287 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 05e988be3b28b57da7eaaf731591b3b5 2024-12-04T02:34:20,291 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnNext/05e988be3b28b57da7eaaf731591b3b5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T02:34:20,291 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 05e988be3b28b57da7eaaf731591b3b5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67015461, jitterRate=-0.0013918131589889526}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T02:34:20,292 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 05e988be3b28b57da7eaaf731591b3b5 2024-12-04T02:34:20,293 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 05e988be3b28b57da7eaaf731591b3b5: Running coprocessor pre-open hook at 1733279660276Writing region info on filesystem at 1733279660276Initializing all the Stores at 1733279660278 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733279660278Cleaning up temporary data from old regions at 1733279660285 (+7 ms)Running coprocessor post-open hooks at 1733279660292 (+7 ms)Region opened successfully at 1733279660292 2024-12-04T02:34:20,294 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5., pid=24, masterSystemTime=1733279660266 2024-12-04T02:34:20,296 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5. 2024-12-04T02:34:20,296 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5. 2024-12-04T02:34:20,297 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=05e988be3b28b57da7eaaf731591b3b5, regionState=OPEN, openSeqNum=2, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:34:20,299 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 05e988be3b28b57da7eaaf731591b3b5, server=40249addbad9,43941,1733279590307 because future has completed 2024-12-04T02:34:20,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-04T02:34:20,305 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 05e988be3b28b57da7eaaf731591b3b5, server=40249addbad9,43941,1733279590307 in 188 msec 2024-12-04T02:34:20,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-04T02:34:20,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=05e988be3b28b57da7eaaf731591b3b5, ASSIGN in 351 msec 2024-12-04T02:34:20,310 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T02:34:20,310 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733279660310"}]},"ts":"1733279660310"} 2024-12-04T02:34:20,312 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-04T02:34:20,313 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T02:34:20,315 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2030 sec 2024-12-04T02:34:21,296 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T02:34:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-04T02:34:22,130 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-12-04T02:34:22,130 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-04T02:34:22,130 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T02:34:22,136 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-04T02:34:22,137 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T02:34:22,137 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-12-04T02:34:22,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5., hostname=40249addbad9,43941,1733279590307, seqNum=2] 2024-12-04T02:34:22,150 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-04T02:34:22,151 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-04T02:34:22,154 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-04T02:34:22,156 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-04T02:34:22,157 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-04T02:34:22,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-04T02:34:24,230 INFO [master/40249addbad9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-04T02:34:24,230 INFO [master/40249addbad9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-04T02:34:25,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43941 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '3447756289302593580' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-04T02:34:25,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43941 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:39822 deadline: 1733279666162, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '3447756289302593580' does not exist 2024-12-04T02:34:25,163 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 40249addbad9,43941,1733279590307 for closing scanner id = 3447756289302593580 for 05e988be3b28b57da7eaaf731591b3b5 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '3447756289302593580' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T02:34:25,169 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 223), OpenFileDescriptor=448 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=313 (was 305) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4318 (was 4319) 2024-12-04T02:34:25,177 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=313, ProcessCount=11, AvailableMemoryMB=4318 2024-12-04T02:34:25,179 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43601 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T02:34:25,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43601 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-12-04T02:34:25,182 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T02:34:25,182 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:34:25,182 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43601 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-12-04T02:34:25,184 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T02:34:25,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T02:34:25,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741849_1025 (size=419) 2024-12-04T02:34:25,451 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-12-04T02:34:25,597 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c843721ad153bda320efe8a6bbfce0a4, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7 2024-12-04T02:34:25,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741850_1026 (size=70) 2024-12-04T02:34:25,609 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:34:25,609 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing c843721ad153bda320efe8a6bbfce0a4, disabling compactions & flushes 2024-12-04T02:34:25,609 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4. 2024-12-04T02:34:25,609 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4. 2024-12-04T02:34:25,609 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4. after waiting 0 ms 2024-12-04T02:34:25,609 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4. 2024-12-04T02:34:25,609 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4. 2024-12-04T02:34:25,609 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for c843721ad153bda320efe8a6bbfce0a4: Waiting for close lock at 1733279665609Disabling compacts and flushes for region at 1733279665609Disabling writes for close at 1733279665609Writing region close event to WAL at 1733279665609Closed at 1733279665609 2024-12-04T02:34:25,611 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T02:34:25,611 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1733279665611"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733279665611"}]},"ts":"1733279665611"} 2024-12-04T02:34:25,614 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T02:34:25,616 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T02:34:25,616 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733279665616"}]},"ts":"1733279665616"} 2024-12-04T02:34:25,619 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-04T02:34:25,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=c843721ad153bda320efe8a6bbfce0a4, ASSIGN}] 2024-12-04T02:34:25,621 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=c843721ad153bda320efe8a6bbfce0a4, ASSIGN 2024-12-04T02:34:25,622 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=c843721ad153bda320efe8a6bbfce0a4, ASSIGN; state=OFFLINE, location=40249addbad9,43941,1733279590307; forceNewPlan=false, retain=false 2024-12-04T02:34:25,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-12-04T02:34:25,773 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=c843721ad153bda320efe8a6bbfce0a4, regionState=OPENING, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:34:25,778 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=c843721ad153bda320efe8a6bbfce0a4, ASSIGN because future has completed 2024-12-04T02:34:25,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure c843721ad153bda320efe8a6bbfce0a4, server=40249addbad9,43941,1733279590307}] 2024-12-04T02:34:25,939 INFO [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4. 2024-12-04T02:34:25,939 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => c843721ad153bda320efe8a6bbfce0a4, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4.', STARTKEY => '', ENDKEY => ''} 2024-12-04T02:34:25,940 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync c843721ad153bda320efe8a6bbfce0a4 2024-12-04T02:34:25,940 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:34:25,940 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for c843721ad153bda320efe8a6bbfce0a4 2024-12-04T02:34:25,940 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for c843721ad153bda320efe8a6bbfce0a4 2024-12-04T02:34:25,942 INFO [StoreOpener-c843721ad153bda320efe8a6bbfce0a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c843721ad153bda320efe8a6bbfce0a4 2024-12-04T02:34:25,945 INFO [StoreOpener-c843721ad153bda320efe8a6bbfce0a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c843721ad153bda320efe8a6bbfce0a4 columnFamilyName testFamily 2024-12-04T02:34:25,945 DEBUG [StoreOpener-c843721ad153bda320efe8a6bbfce0a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:34:25,945 INFO [StoreOpener-c843721ad153bda320efe8a6bbfce0a4-1 {}] regionserver.HStore(327): Store=c843721ad153bda320efe8a6bbfce0a4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T02:34:25,946 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for c843721ad153bda320efe8a6bbfce0a4 2024-12-04T02:34:25,947 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/c843721ad153bda320efe8a6bbfce0a4 2024-12-04T02:34:25,948 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/c843721ad153bda320efe8a6bbfce0a4 2024-12-04T02:34:25,948 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for c843721ad153bda320efe8a6bbfce0a4 2024-12-04T02:34:25,949 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for c843721ad153bda320efe8a6bbfce0a4 2024-12-04T02:34:25,951 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for c843721ad153bda320efe8a6bbfce0a4 2024-12-04T02:34:25,953 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/c843721ad153bda320efe8a6bbfce0a4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T02:34:25,954 INFO [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened c843721ad153bda320efe8a6bbfce0a4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69196393, jitterRate=0.031106606125831604}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T02:34:25,954 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c843721ad153bda320efe8a6bbfce0a4 2024-12-04T02:34:25,955 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for c843721ad153bda320efe8a6bbfce0a4: Running coprocessor pre-open hook at 1733279665940Writing region info on filesystem at 1733279665940Initializing all the Stores at 1733279665942 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733279665942Cleaning up temporary data from old regions at 1733279665949 (+7 ms)Running coprocessor post-open hooks at 1733279665954 (+5 ms)Region opened successfully at 1733279665955 (+1 ms) 2024-12-04T02:34:25,956 INFO [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4., pid=27, masterSystemTime=1733279665933 2024-12-04T02:34:25,959 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4. 2024-12-04T02:34:25,959 INFO [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4. 2024-12-04T02:34:25,959 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=c843721ad153bda320efe8a6bbfce0a4, regionState=OPEN, openSeqNum=2, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:34:25,962 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure c843721ad153bda320efe8a6bbfce0a4, server=40249addbad9,43941,1733279590307 because future has completed 2024-12-04T02:34:25,965 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-04T02:34:25,965 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure c843721ad153bda320efe8a6bbfce0a4, server=40249addbad9,43941,1733279590307 in 185 msec 2024-12-04T02:34:25,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-04T02:34:25,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=c843721ad153bda320efe8a6bbfce0a4, ASSIGN in 346 msec 2024-12-04T02:34:25,968 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T02:34:25,968 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733279665968"}]},"ts":"1733279665968"} 2024-12-04T02:34:25,970 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-04T02:34:25,972 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T02:34:25,974 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 793 msec 2024-12-04T02:34:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-04T02:34:26,190 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-12-04T02:34:26,190 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-04T02:34:26,190 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T02:34:26,199 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-04T02:34:26,199 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T02:34:26,199 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-12-04T02:34:26,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4., hostname=40249addbad9,43941,1733279590307, seqNum=2] 2024-12-04T02:34:26,215 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-04T02:34:26,215 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-04T02:34:26,218 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-04T02:34:26,221 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43941 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-04T02:34:29,900 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-04T02:34:29,900 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-04T02:34:29,901 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-04T02:34:29,901 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-04T02:34:31,442 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T02:34:31,472 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-12-04T02:34:32,228 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4., hostname=40249addbad9,43941,1733279590307, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4., hostname=40249addbad9,43941,1733279590307, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=40249addbad9:43941 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6009ms, rpcTimeout=5999ms 2024-12-04T02:34:32,229 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4., hostname=40249addbad9,43941,1733279590307, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6009ms, rpcTimeout=5999ms 2024-12-04T02:34:32,229 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4., hostname=40249addbad9,43941,1733279590307, seqNum=2 from cache 2024-12-04T02:34:32,237 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=326 (was 313) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4301 (was 4318) 2024-12-04T02:34:32,245 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=326, ProcessCount=11, AvailableMemoryMB=4300 2024-12-04T02:34:32,248 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43601 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T02:34:32,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43601 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-12-04T02:34:32,250 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T02:34:32,251 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:34:32,251 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43601 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-12-04T02:34:32,252 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T02:34:32,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-04T02:34:32,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741851_1027 (size=407) 2024-12-04T02:34:32,263 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 17a7726b52c9202a3d40b73e80eac95a, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7 2024-12-04T02:34:32,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741852_1028 (size=58) 2024-12-04T02:34:32,673 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:34:32,674 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 17a7726b52c9202a3d40b73e80eac95a, disabling compactions & flushes 2024-12-04T02:34:32,674 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a. 2024-12-04T02:34:32,674 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a. 2024-12-04T02:34:32,674 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a. after waiting 0 ms 2024-12-04T02:34:32,675 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a. 2024-12-04T02:34:32,675 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a. 2024-12-04T02:34:32,675 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 17a7726b52c9202a3d40b73e80eac95a: Waiting for close lock at 1733279672673Disabling compacts and flushes for region at 1733279672673Disabling writes for close at 1733279672674 (+1 ms)Writing region close event to WAL at 1733279672675 (+1 ms)Closed at 1733279672675 2024-12-04T02:34:32,678 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T02:34:32,678 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1733279672678"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733279672678"}]},"ts":"1733279672678"} 2024-12-04T02:34:32,681 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T02:34:32,683 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T02:34:32,683 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733279672683"}]},"ts":"1733279672683"} 2024-12-04T02:34:32,685 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-04T02:34:32,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=17a7726b52c9202a3d40b73e80eac95a, ASSIGN}] 2024-12-04T02:34:32,687 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=17a7726b52c9202a3d40b73e80eac95a, ASSIGN 2024-12-04T02:34:32,689 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=17a7726b52c9202a3d40b73e80eac95a, ASSIGN; state=OFFLINE, location=40249addbad9,43941,1733279590307; forceNewPlan=false, retain=false 2024-12-04T02:34:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-12-04T02:34:32,840 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=17a7726b52c9202a3d40b73e80eac95a, regionState=OPENING, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:34:32,843 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=17a7726b52c9202a3d40b73e80eac95a, ASSIGN because future has completed 2024-12-04T02:34:32,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17a7726b52c9202a3d40b73e80eac95a, server=40249addbad9,43941,1733279590307}] 2024-12-04T02:34:33,002 INFO [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a. 2024-12-04T02:34:33,002 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 17a7726b52c9202a3d40b73e80eac95a, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a.', STARTKEY => '', ENDKEY => ''} 2024-12-04T02:34:33,003 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 17a7726b52c9202a3d40b73e80eac95a 2024-12-04T02:34:33,003 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:34:33,003 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 17a7726b52c9202a3d40b73e80eac95a 2024-12-04T02:34:33,003 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 17a7726b52c9202a3d40b73e80eac95a 2024-12-04T02:34:33,005 INFO [StoreOpener-17a7726b52c9202a3d40b73e80eac95a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 17a7726b52c9202a3d40b73e80eac95a 2024-12-04T02:34:33,007 INFO [StoreOpener-17a7726b52c9202a3d40b73e80eac95a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17a7726b52c9202a3d40b73e80eac95a columnFamilyName testFamily 2024-12-04T02:34:33,007 DEBUG [StoreOpener-17a7726b52c9202a3d40b73e80eac95a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:34:33,008 INFO [StoreOpener-17a7726b52c9202a3d40b73e80eac95a-1 {}] regionserver.HStore(327): Store=17a7726b52c9202a3d40b73e80eac95a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T02:34:33,008 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 17a7726b52c9202a3d40b73e80eac95a 2024-12-04T02:34:33,009 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnNext/17a7726b52c9202a3d40b73e80eac95a 2024-12-04T02:34:33,010 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnNext/17a7726b52c9202a3d40b73e80eac95a 2024-12-04T02:34:33,011 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 17a7726b52c9202a3d40b73e80eac95a 2024-12-04T02:34:33,011 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 17a7726b52c9202a3d40b73e80eac95a 2024-12-04T02:34:33,013 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 17a7726b52c9202a3d40b73e80eac95a 2024-12-04T02:34:33,017 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnNext/17a7726b52c9202a3d40b73e80eac95a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T02:34:33,018 INFO [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 17a7726b52c9202a3d40b73e80eac95a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66989860, jitterRate=-0.0017732977867126465}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T02:34:33,018 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 17a7726b52c9202a3d40b73e80eac95a 2024-12-04T02:34:33,018 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 17a7726b52c9202a3d40b73e80eac95a: Running coprocessor pre-open hook at 1733279673003Writing region info on filesystem at 1733279673003Initializing all the Stores at 1733279673005 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733279673005Cleaning up temporary data from old regions at 1733279673011 (+6 ms)Running coprocessor post-open hooks at 1733279673018 (+7 ms)Region opened successfully at 1733279673018 2024-12-04T02:34:33,020 INFO [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a., pid=30, masterSystemTime=1733279672996 2024-12-04T02:34:33,023 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a. 2024-12-04T02:34:33,023 INFO [RS_OPEN_PRIORITY_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a. 2024-12-04T02:34:33,024 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=17a7726b52c9202a3d40b73e80eac95a, regionState=OPEN, openSeqNum=2, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:34:33,026 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17a7726b52c9202a3d40b73e80eac95a, server=40249addbad9,43941,1733279590307 because future has completed 2024-12-04T02:34:33,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-04T02:34:33,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 17a7726b52c9202a3d40b73e80eac95a, server=40249addbad9,43941,1733279590307 in 185 msec 2024-12-04T02:34:33,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-04T02:34:33,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=17a7726b52c9202a3d40b73e80eac95a, ASSIGN in 345 msec 2024-12-04T02:34:33,035 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T02:34:33,035 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733279673035"}]},"ts":"1733279673035"} 2024-12-04T02:34:33,038 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-04T02:34:33,039 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T02:34:33,043 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 791 msec 2024-12-04T02:34:33,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-04T02:34:33,259 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-12-04T02:34:33,259 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-04T02:34:33,259 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T02:34:33,264 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-04T02:34:33,265 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T02:34:33,265 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-12-04T02:34:33,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a., hostname=40249addbad9,43941,1733279590307, seqNum=2] 2024-12-04T02:34:33,280 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-04T02:34:33,281 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-04T02:34:33,284 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-04T02:34:33,287 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-04T02:34:33,288 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-04T02:34:33,289 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43941 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-04T02:34:37,473 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-12-04T02:34:38,391 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T02:34:38,507 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T02:34:39,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43941 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '3447756289302593588' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-04T02:34:39,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43941 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:39822 deadline: 1733279685295, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '3447756289302593588' does not exist 2024-12-04T02:34:39,297 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 40249addbad9,43941,1733279590307 for closing scanner id = 3447756289302593588 for 17a7726b52c9202a3d40b73e80eac95a of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '3447756289302593588' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T02:34:39,302 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=300 (was 326), ProcessCount=11 (was 11), AvailableMemoryMB=4253 (was 4300) 2024-12-04T02:34:39,311 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=300, ProcessCount=11, AvailableMemoryMB=4253 2024-12-04T02:34:39,313 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T02:34:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-12-04T02:34:39,317 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T02:34:39,317 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:34:39,317 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-12-04T02:34:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-04T02:34:39,319 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T02:34:39,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741853_1029 (size=423) 2024-12-04T02:34:39,731 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 66b379fc7c2902adaf1f117fe455c1d2, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7 2024-12-04T02:34:39,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741854_1030 (size=74) 2024-12-04T02:34:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-12-04T02:34:39,900 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-04T02:34:39,900 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-04T02:34:40,141 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:34:40,141 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 66b379fc7c2902adaf1f117fe455c1d2, disabling compactions & flushes 2024-12-04T02:34:40,142 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2. 2024-12-04T02:34:40,142 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2. 2024-12-04T02:34:40,142 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2. after waiting 0 ms 2024-12-04T02:34:40,142 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2. 2024-12-04T02:34:40,142 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2. 2024-12-04T02:34:40,142 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 66b379fc7c2902adaf1f117fe455c1d2: Waiting for close lock at 1733279680141Disabling compacts and flushes for region at 1733279680141Disabling writes for close at 1733279680142 (+1 ms)Writing region close event to WAL at 1733279680142Closed at 1733279680142 2024-12-04T02:34:40,144 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T02:34:40,145 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733279680145"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733279680145"}]},"ts":"1733279680145"} 2024-12-04T02:34:40,150 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T02:34:40,152 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T02:34:40,153 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733279680152"}]},"ts":"1733279680152"} 2024-12-04T02:34:40,156 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-04T02:34:40,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=66b379fc7c2902adaf1f117fe455c1d2, ASSIGN}] 2024-12-04T02:34:40,158 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=66b379fc7c2902adaf1f117fe455c1d2, ASSIGN 2024-12-04T02:34:40,159 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=66b379fc7c2902adaf1f117fe455c1d2, ASSIGN; state=OFFLINE, location=40249addbad9,43941,1733279590307; forceNewPlan=false, retain=false 2024-12-04T02:34:40,310 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=66b379fc7c2902adaf1f117fe455c1d2, regionState=OPENING, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:34:40,314 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=66b379fc7c2902adaf1f117fe455c1d2, ASSIGN because future has completed 2024-12-04T02:34:40,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66b379fc7c2902adaf1f117fe455c1d2, server=40249addbad9,43941,1733279590307}] 2024-12-04T02:34:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-04T02:34:40,475 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2. 2024-12-04T02:34:40,475 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 66b379fc7c2902adaf1f117fe455c1d2, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2.', STARTKEY => '', ENDKEY => ''} 2024-12-04T02:34:40,476 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 66b379fc7c2902adaf1f117fe455c1d2 2024-12-04T02:34:40,476 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T02:34:40,476 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 66b379fc7c2902adaf1f117fe455c1d2 2024-12-04T02:34:40,476 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 66b379fc7c2902adaf1f117fe455c1d2 2024-12-04T02:34:40,482 INFO [StoreOpener-66b379fc7c2902adaf1f117fe455c1d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 66b379fc7c2902adaf1f117fe455c1d2 2024-12-04T02:34:40,484 INFO [StoreOpener-66b379fc7c2902adaf1f117fe455c1d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66b379fc7c2902adaf1f117fe455c1d2 columnFamilyName testFamily 2024-12-04T02:34:40,484 DEBUG [StoreOpener-66b379fc7c2902adaf1f117fe455c1d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T02:34:40,488 INFO [StoreOpener-66b379fc7c2902adaf1f117fe455c1d2-1 {}] regionserver.HStore(327): Store=66b379fc7c2902adaf1f117fe455c1d2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T02:34:40,488 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 66b379fc7c2902adaf1f117fe455c1d2 2024-12-04T02:34:40,490 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnOpenScannerAsync/66b379fc7c2902adaf1f117fe455c1d2 2024-12-04T02:34:40,490 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnOpenScannerAsync/66b379fc7c2902adaf1f117fe455c1d2 2024-12-04T02:34:40,491 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 66b379fc7c2902adaf1f117fe455c1d2 2024-12-04T02:34:40,491 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 66b379fc7c2902adaf1f117fe455c1d2 2024-12-04T02:34:40,493 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 66b379fc7c2902adaf1f117fe455c1d2 2024-12-04T02:34:40,496 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnOpenScannerAsync/66b379fc7c2902adaf1f117fe455c1d2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T02:34:40,497 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 66b379fc7c2902adaf1f117fe455c1d2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69548371, jitterRate=0.036351487040519714}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T02:34:40,497 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 66b379fc7c2902adaf1f117fe455c1d2 2024-12-04T02:34:40,497 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 66b379fc7c2902adaf1f117fe455c1d2: Running coprocessor pre-open hook at 1733279680476Writing region info on filesystem at 1733279680476Initializing all the Stores at 1733279680481 (+5 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733279680481Cleaning up temporary data from old regions at 1733279680491 (+10 ms)Running coprocessor post-open hooks at 1733279680497 (+6 ms)Region opened successfully at 1733279680497 2024-12-04T02:34:40,499 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2., pid=33, masterSystemTime=1733279680468 2024-12-04T02:34:40,501 DEBUG [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2. 2024-12-04T02:34:40,501 INFO [RS_OPEN_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2. 2024-12-04T02:34:40,502 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=66b379fc7c2902adaf1f117fe455c1d2, regionState=OPEN, openSeqNum=2, regionLocation=40249addbad9,43941,1733279590307 2024-12-04T02:34:40,504 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66b379fc7c2902adaf1f117fe455c1d2, server=40249addbad9,43941,1733279590307 because future has completed 2024-12-04T02:34:40,508 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-04T02:34:40,508 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 66b379fc7c2902adaf1f117fe455c1d2, server=40249addbad9,43941,1733279590307 in 191 msec 2024-12-04T02:34:40,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-04T02:34:40,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=66b379fc7c2902adaf1f117fe455c1d2, ASSIGN in 352 msec 2024-12-04T02:34:40,511 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T02:34:40,511 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733279680511"}]},"ts":"1733279680511"} 2024-12-04T02:34:40,514 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-04T02:34:40,515 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T02:34:40,516 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.2020 sec 2024-12-04T02:34:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43601 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-04T02:34:42,349 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-12-04T02:34:42,349 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-04T02:34:42,349 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T02:34:42,353 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-04T02:34:42,354 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T02:34:42,354 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-12-04T02:34:42,357 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2., hostname=40249addbad9,43941,1733279590307, seqNum=2] 2024-12-04T02:34:42,373 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-04T02:34:42,374 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-04T02:34:42,378 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-04T02:34:42,381 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-04T02:34:43,388 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2., hostname=40249addbad9,43941,1733279590307, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2., hostname=40249addbad9,43941,1733279590307, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=40249addbad9:43941 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-12-04T02:34:43,388 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2., hostname=40249addbad9,43941,1733279590307, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-12-04T02:34:43,388 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2., hostname=40249addbad9,43941,1733279590307, seqNum=2 from cache 2024-12-04T02:34:43,474 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-12-04T02:34:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-12-04T02:34:44,180 DEBUG [master/40249addbad9:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-04T02:34:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2., hostname=40249addbad9,43941,1733279590307, seqNum=2] 2024-12-04T02:34:44,408 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43941 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-04T02:34:45,388 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2., hostname=40249addbad9,43941,1733279590307, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2., hostname=40249addbad9,43941,1733279590307, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=40249addbad9:43941 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=981ms, rpcTimeout=972ms 2024-12-04T02:34:45,388 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2., hostname=40249addbad9,43941,1733279590307, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=981ms, rpcTimeout=972ms 2024-12-04T02:34:45,388 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2., hostname=40249addbad9,43941,1733279590307, seqNum=2 from cache 2024-12-04T02:34:45,401 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=300 (was 300), ProcessCount=11 (was 11), AvailableMemoryMB=4220 (was 4253) 2024-12-04T02:34:45,402 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T02:34:45,402 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T02:34:45,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T02:34:45,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T02:34:45,409 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T02:34:45,409 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T02:34:45,409 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T02:34:45,409 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T02:34:45,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T02:34:45,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T02:34:45,409 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T02:34:45,409 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T02:34:45,410 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1961867613, stopped=false 2024-12-04T02:34:45,410 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=40249addbad9,43601,1733279589466 2024-12-04T02:34:45,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T02:34:45,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43941-0x1019e2f9c250001, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T02:34:45,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T02:34:45,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43941-0x1019e2f9c250001, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T02:34:45,454 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T02:34:45,456 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43941-0x1019e2f9c250001, quorum=127.0.0.1:53960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T02:34:45,456 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T02:34:45,456 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T02:34:45,456 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T02:34:45,457 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T02:34:45,457 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '40249addbad9,43941,1733279590307' ***** 2024-12-04T02:34:45,457 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T02:34:45,458 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T02:34:45,458 INFO [RS:0;40249addbad9:43941 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T02:34:45,458 INFO [RS:0;40249addbad9:43941 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T02:34:45,458 INFO [RS:0;40249addbad9:43941 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T02:34:45,459 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(3091): Received CLOSE for 46322ad3899db909759a023e550f5dc8 2024-12-04T02:34:45,459 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(3091): Received CLOSE for 17a7726b52c9202a3d40b73e80eac95a 2024-12-04T02:34:45,459 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(3091): Received CLOSE for 66b379fc7c2902adaf1f117fe455c1d2 2024-12-04T02:34:45,460 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(3091): Received CLOSE for c843721ad153bda320efe8a6bbfce0a4 2024-12-04T02:34:45,460 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(3091): Received CLOSE for 358638b58760ba89e229465286e54e93 2024-12-04T02:34:45,460 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(3091): Received CLOSE for a921ec0371ebd57e4fc59d2d75b9cee9 2024-12-04T02:34:45,460 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(3091): Received CLOSE for 53b175aca7affc4c1f7e036710ba4734 2024-12-04T02:34:45,460 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(3091): Received CLOSE for 05e988be3b28b57da7eaaf731591b3b5 2024-12-04T02:34:45,460 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(3091): Received CLOSE for 20d9ebc06c69a540e5cbfa93071b4415 2024-12-04T02:34:45,460 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 46322ad3899db909759a023e550f5dc8, disabling compactions & flushes 2024-12-04T02:34:45,460 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(3091): Received CLOSE for be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:34:45,460 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8. 2024-12-04T02:34:45,460 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8. 2024-12-04T02:34:45,460 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(959): stopping server 40249addbad9,43941,1733279590307 2024-12-04T02:34:45,460 INFO [RS:0;40249addbad9:43941 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T02:34:45,460 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8. after waiting 0 ms 2024-12-04T02:34:45,460 INFO [RS:0;40249addbad9:43941 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;40249addbad9:43941. 2024-12-04T02:34:45,460 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8. 2024-12-04T02:34:45,461 DEBUG [RS:0;40249addbad9:43941 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T02:34:45,461 DEBUG [RS:0;40249addbad9:43941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T02:34:45,461 INFO [RS:0;40249addbad9:43941 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T02:34:45,461 INFO [RS:0;40249addbad9:43941 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T02:34:45,461 INFO [RS:0;40249addbad9:43941 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T02:34:45,461 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T02:34:45,462 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-12-04T02:34:45,462 DEBUG [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T02:34:45,462 INFO [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T02:34:45,462 DEBUG [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T02:34:45,462 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on 05e988be3b28b57da7eaaf731591b3b5, 1588230740, 17a7726b52c9202a3d40b73e80eac95a, 20d9ebc06c69a540e5cbfa93071b4415, 358638b58760ba89e229465286e54e93, 46322ad3899db909759a023e550f5dc8, 53b175aca7affc4c1f7e036710ba4734, 66b379fc7c2902adaf1f117fe455c1d2, a921ec0371ebd57e4fc59d2d75b9cee9, be4fba22bce0f0993f9e6bdd23cbac06, c843721ad153bda320efe8a6bbfce0a4 2024-12-04T02:34:45,462 DEBUG [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T02:34:45,462 DEBUG [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T02:34:45,464 INFO [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-12-04T02:34:45,464 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 46322ad3899db909759a023e550f5dc8 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-04T02:34:45,521 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnOpenScanner/46322ad3899db909759a023e550f5dc8/.tmp/testFamily/750f6705cd6544439a49fa68c2cc7294 is 61, key is row-0/testFamily:testQualifier/1733279630660/Put/seqid=0 2024-12-04T02:34:45,524 DEBUG [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740/.tmp/info/61d2255254ea48f7b1e6824cd35a65fe is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93./info:regioninfo/1733279625672/Put/seqid=0 2024-12-04T02:34:45,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741856_1032 (size=5049) 2024-12-04T02:34:45,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741855_1031 (size=20564) 2024-12-04T02:34:45,531 INFO [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740/.tmp/info/61d2255254ea48f7b1e6824cd35a65fe 2024-12-04T02:34:45,581 DEBUG [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740/.tmp/ns/d39b374891e244d6829aa3c5931d6947 is 43, key is default/ns:d/1733279624126/Put/seqid=0 2024-12-04T02:34:45,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741857_1033 (size=5153) 2024-12-04T02:34:45,662 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on 05e988be3b28b57da7eaaf731591b3b5, 1588230740, 17a7726b52c9202a3d40b73e80eac95a, 20d9ebc06c69a540e5cbfa93071b4415, 358638b58760ba89e229465286e54e93, 46322ad3899db909759a023e550f5dc8, 53b175aca7affc4c1f7e036710ba4734, 66b379fc7c2902adaf1f117fe455c1d2, a921ec0371ebd57e4fc59d2d75b9cee9, be4fba22bce0f0993f9e6bdd23cbac06, c843721ad153bda320efe8a6bbfce0a4 2024-12-04T02:34:45,863 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on 05e988be3b28b57da7eaaf731591b3b5, 1588230740, 17a7726b52c9202a3d40b73e80eac95a, 20d9ebc06c69a540e5cbfa93071b4415, 358638b58760ba89e229465286e54e93, 46322ad3899db909759a023e550f5dc8, 53b175aca7affc4c1f7e036710ba4734, 66b379fc7c2902adaf1f117fe455c1d2, a921ec0371ebd57e4fc59d2d75b9cee9, be4fba22bce0f0993f9e6bdd23cbac06, c843721ad153bda320efe8a6bbfce0a4 2024-12-04T02:34:45,909 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-04T02:34:45,932 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnOpenScanner/46322ad3899db909759a023e550f5dc8/.tmp/testFamily/750f6705cd6544439a49fa68c2cc7294 2024-12-04T02:34:45,944 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnOpenScanner/46322ad3899db909759a023e550f5dc8/.tmp/testFamily/750f6705cd6544439a49fa68c2cc7294 as hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnOpenScanner/46322ad3899db909759a023e550f5dc8/testFamily/750f6705cd6544439a49fa68c2cc7294 2024-12-04T02:34:45,954 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnOpenScanner/46322ad3899db909759a023e550f5dc8/testFamily/750f6705cd6544439a49fa68c2cc7294, entries=4, sequenceid=8, filesize=4.9 K 2024-12-04T02:34:45,965 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 46322ad3899db909759a023e550f5dc8 in 498ms, sequenceid=8, compaction requested=false 2024-12-04T02:34:45,973 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnOpenScanner/46322ad3899db909759a023e550f5dc8/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-04T02:34:45,976 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8. 2024-12-04T02:34:45,976 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 46322ad3899db909759a023e550f5dc8: Waiting for close lock at 1733279685460Running coprocessor pre-close hooks at 1733279685460Disabling compacts and flushes for region at 1733279685460Disabling writes for close at 1733279685460Obtaining lock to block concurrent updates at 1733279685465 (+5 ms)Preparing flush snapshotting stores in 46322ad3899db909759a023e550f5dc8 at 1733279685465Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733279685472 (+7 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8. at 1733279685474 (+2 ms)Flushing 46322ad3899db909759a023e550f5dc8/testFamily: creating writer at 1733279685476 (+2 ms)Flushing 46322ad3899db909759a023e550f5dc8/testFamily: appending metadata at 1733279685514 (+38 ms)Flushing 46322ad3899db909759a023e550f5dc8/testFamily: closing flushed file at 1733279685517 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b719b40: reopening flushed file at 1733279685942 (+425 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 46322ad3899db909759a023e550f5dc8 in 498ms, sequenceid=8, compaction requested=false at 1733279685965 (+23 ms)Writing region close event to WAL at 1733279685967 (+2 ms)Running coprocessor post-close hooks at 1733279685974 (+7 ms)Closed at 1733279685976 (+2 ms) 2024-12-04T02:34:45,976 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733279627591.46322ad3899db909759a023e550f5dc8. 2024-12-04T02:34:45,977 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 17a7726b52c9202a3d40b73e80eac95a, disabling compactions & flushes 2024-12-04T02:34:45,977 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a. 2024-12-04T02:34:45,977 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a. 2024-12-04T02:34:45,977 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a. after waiting 0 ms 2024-12-04T02:34:45,977 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a. 2024-12-04T02:34:45,977 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 17a7726b52c9202a3d40b73e80eac95a 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-04T02:34:45,978 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T02:34:45,987 INFO [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740/.tmp/ns/d39b374891e244d6829aa3c5931d6947 2024-12-04T02:34:45,999 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnNext/17a7726b52c9202a3d40b73e80eac95a/.tmp/testFamily/68d5490a227642a587e9ea5c16604813 is 61, key is row-0/testFamily:testQualifier/1733279673269/Put/seqid=0 2024-12-04T02:34:46,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741858_1034 (size=5049) 2024-12-04T02:34:46,007 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnNext/17a7726b52c9202a3d40b73e80eac95a/.tmp/testFamily/68d5490a227642a587e9ea5c16604813 2024-12-04T02:34:46,016 DEBUG [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740/.tmp/table/899dc07dce13482e9a2cb192cc4206d7 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1733279625692/Put/seqid=0 2024-12-04T02:34:46,017 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnNext/17a7726b52c9202a3d40b73e80eac95a/.tmp/testFamily/68d5490a227642a587e9ea5c16604813 as hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnNext/17a7726b52c9202a3d40b73e80eac95a/testFamily/68d5490a227642a587e9ea5c16604813 2024-12-04T02:34:46,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741859_1035 (size=6699) 2024-12-04T02:34:46,026 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnNext/17a7726b52c9202a3d40b73e80eac95a/testFamily/68d5490a227642a587e9ea5c16604813, entries=4, sequenceid=8, filesize=4.9 K 2024-12-04T02:34:46,027 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 17a7726b52c9202a3d40b73e80eac95a in 50ms, sequenceid=8, compaction requested=false 2024-12-04T02:34:46,033 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnNext/17a7726b52c9202a3d40b73e80eac95a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-04T02:34:46,033 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a. 2024-12-04T02:34:46,034 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 17a7726b52c9202a3d40b73e80eac95a: Waiting for close lock at 1733279685977Running coprocessor pre-close hooks at 1733279685977Disabling compacts and flushes for region at 1733279685977Disabling writes for close at 1733279685977Obtaining lock to block concurrent updates at 1733279685977Preparing flush snapshotting stores in 17a7726b52c9202a3d40b73e80eac95a at 1733279685977Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733279685977Flushing stores of hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a. at 1733279685978 (+1 ms)Flushing 17a7726b52c9202a3d40b73e80eac95a/testFamily: creating writer at 1733279685978Flushing 17a7726b52c9202a3d40b73e80eac95a/testFamily: appending metadata at 1733279685999 (+21 ms)Flushing 17a7726b52c9202a3d40b73e80eac95a/testFamily: closing flushed file at 1733279685999Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ad5e5a1: reopening flushed file at 1733279686014 (+15 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 17a7726b52c9202a3d40b73e80eac95a in 50ms, sequenceid=8, compaction requested=false at 1733279686027 (+13 ms)Writing region close event to WAL at 1733279686028 (+1 ms)Running coprocessor post-close hooks at 1733279686033 (+5 ms)Closed at 1733279686033 2024-12-04T02:34:46,034 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1733279672247.17a7726b52c9202a3d40b73e80eac95a. 2024-12-04T02:34:46,034 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 66b379fc7c2902adaf1f117fe455c1d2, disabling compactions & flushes 2024-12-04T02:34:46,034 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2. 2024-12-04T02:34:46,034 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2. 2024-12-04T02:34:46,034 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2. after waiting 0 ms 2024-12-04T02:34:46,034 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2. 2024-12-04T02:34:46,034 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 66b379fc7c2902adaf1f117fe455c1d2 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-04T02:34:46,050 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnOpenScannerAsync/66b379fc7c2902adaf1f117fe455c1d2/.tmp/testFamily/1e2c23ed417746b1821434d0de9ada01 is 61, key is row-0/testFamily:testQualifier/1733279682359/Put/seqid=0 2024-12-04T02:34:46,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741860_1036 (size=5049) 2024-12-04T02:34:46,063 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on 05e988be3b28b57da7eaaf731591b3b5, 1588230740, 20d9ebc06c69a540e5cbfa93071b4415, 358638b58760ba89e229465286e54e93, 53b175aca7affc4c1f7e036710ba4734, 66b379fc7c2902adaf1f117fe455c1d2, a921ec0371ebd57e4fc59d2d75b9cee9, be4fba22bce0f0993f9e6bdd23cbac06, c843721ad153bda320efe8a6bbfce0a4 2024-12-04T02:34:46,263 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on 05e988be3b28b57da7eaaf731591b3b5, 1588230740, 20d9ebc06c69a540e5cbfa93071b4415, 358638b58760ba89e229465286e54e93, 53b175aca7affc4c1f7e036710ba4734, 66b379fc7c2902adaf1f117fe455c1d2, a921ec0371ebd57e4fc59d2d75b9cee9, be4fba22bce0f0993f9e6bdd23cbac06, c843721ad153bda320efe8a6bbfce0a4 2024-12-04T02:34:46,348 INFO [regionserver/40249addbad9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T02:34:46,348 INFO [regionserver/40249addbad9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T02:34:46,428 INFO [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740/.tmp/table/899dc07dce13482e9a2cb192cc4206d7 2024-12-04T02:34:46,443 DEBUG [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740/.tmp/info/61d2255254ea48f7b1e6824cd35a65fe as hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740/info/61d2255254ea48f7b1e6824cd35a65fe 2024-12-04T02:34:46,453 INFO [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740/info/61d2255254ea48f7b1e6824cd35a65fe, entries=100, sequenceid=56, filesize=20.1 K 2024-12-04T02:34:46,455 DEBUG [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740/.tmp/ns/d39b374891e244d6829aa3c5931d6947 as hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740/ns/d39b374891e244d6829aa3c5931d6947 2024-12-04T02:34:46,455 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnOpenScannerAsync/66b379fc7c2902adaf1f117fe455c1d2/.tmp/testFamily/1e2c23ed417746b1821434d0de9ada01 2024-12-04T02:34:46,462 INFO [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740/ns/d39b374891e244d6829aa3c5931d6947, entries=2, sequenceid=56, filesize=5.0 K 2024-12-04T02:34:46,463 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnOpenScannerAsync/66b379fc7c2902adaf1f117fe455c1d2/.tmp/testFamily/1e2c23ed417746b1821434d0de9ada01 as hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnOpenScannerAsync/66b379fc7c2902adaf1f117fe455c1d2/testFamily/1e2c23ed417746b1821434d0de9ada01 2024-12-04T02:34:46,463 DEBUG [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740/.tmp/table/899dc07dce13482e9a2cb192cc4206d7 as hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740/table/899dc07dce13482e9a2cb192cc4206d7 2024-12-04T02:34:46,464 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-12-04T02:34:46,464 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1325): Online Regions={66b379fc7c2902adaf1f117fe455c1d2=testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2., c843721ad153bda320efe8a6bbfce0a4=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4., 358638b58760ba89e229465286e54e93=testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93., 1588230740=hbase:meta,,1.1588230740, a921ec0371ebd57e4fc59d2d75b9cee9=testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9., 53b175aca7affc4c1f7e036710ba4734=testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734., 05e988be3b28b57da7eaaf731591b3b5=testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5., 20d9ebc06c69a540e5cbfa93071b4415=testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415., be4fba22bce0f0993f9e6bdd23cbac06=hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06.} 2024-12-04T02:34:46,464 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on 05e988be3b28b57da7eaaf731591b3b5, 1588230740, 20d9ebc06c69a540e5cbfa93071b4415, 358638b58760ba89e229465286e54e93, 53b175aca7affc4c1f7e036710ba4734, 66b379fc7c2902adaf1f117fe455c1d2, a921ec0371ebd57e4fc59d2d75b9cee9, be4fba22bce0f0993f9e6bdd23cbac06, c843721ad153bda320efe8a6bbfce0a4 2024-12-04T02:34:46,472 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnOpenScannerAsync/66b379fc7c2902adaf1f117fe455c1d2/testFamily/1e2c23ed417746b1821434d0de9ada01, entries=4, sequenceid=8, filesize=4.9 K 2024-12-04T02:34:46,472 INFO [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740/table/899dc07dce13482e9a2cb192cc4206d7, entries=20, sequenceid=56, filesize=6.5 K 2024-12-04T02:34:46,473 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 66b379fc7c2902adaf1f117fe455c1d2 in 439ms, sequenceid=8, compaction requested=false 2024-12-04T02:34:46,473 INFO [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1011ms, sequenceid=56, compaction requested=false 2024-12-04T02:34:46,482 DEBUG [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-12-04T02:34:46,482 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnOpenScannerAsync/66b379fc7c2902adaf1f117fe455c1d2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-04T02:34:46,483 DEBUG [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T02:34:46,483 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2. 2024-12-04T02:34:46,483 INFO [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T02:34:46,483 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 66b379fc7c2902adaf1f117fe455c1d2: Waiting for close lock at 1733279686034Running coprocessor pre-close hooks at 1733279686034Disabling compacts and flushes for region at 1733279686034Disabling writes for close at 1733279686034Obtaining lock to block concurrent updates at 1733279686034Preparing flush snapshotting stores in 66b379fc7c2902adaf1f117fe455c1d2 at 1733279686034Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733279686034Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2. at 1733279686035 (+1 ms)Flushing 66b379fc7c2902adaf1f117fe455c1d2/testFamily: creating writer at 1733279686035Flushing 66b379fc7c2902adaf1f117fe455c1d2/testFamily: appending metadata at 1733279686049 (+14 ms)Flushing 66b379fc7c2902adaf1f117fe455c1d2/testFamily: closing flushed file at 1733279686049Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12cfa18: reopening flushed file at 1733279686461 (+412 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 66b379fc7c2902adaf1f117fe455c1d2 in 439ms, sequenceid=8, compaction requested=false at 1733279686473 (+12 ms)Writing region close event to WAL at 1733279686476 (+3 ms)Running coprocessor post-close hooks at 1733279686482 (+6 ms)Closed at 1733279686483 (+1 ms) 2024-12-04T02:34:46,483 DEBUG [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733279685462Running coprocessor pre-close hooks at 1733279685462Disabling compacts and flushes for region at 1733279685462Disabling writes for close at 1733279685462Obtaining lock to block concurrent updates at 1733279685465 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1733279685465Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1733279685472 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733279685474 (+2 ms)Flushing 1588230740/info: creating writer at 1733279685476 (+2 ms)Flushing 1588230740/info: appending metadata at 1733279685521 (+45 ms)Flushing 1588230740/info: closing flushed file at 1733279685521Flushing 1588230740/ns: creating writer at 1733279685566 (+45 ms)Flushing 1588230740/ns: appending metadata at 1733279685580 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733279685580Flushing 1588230740/table: creating writer at 1733279685996 (+416 ms)Flushing 1588230740/table: appending metadata at 1733279686016 (+20 ms)Flushing 1588230740/table: closing flushed file at 1733279686016Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44994cf1: reopening flushed file at 1733279686442 (+426 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cd5b651: reopening flushed file at 1733279686453 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d2a347c: reopening flushed file at 1733279686462 (+9 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1011ms, sequenceid=56, compaction requested=false at 1733279686473 (+11 ms)Writing region close event to WAL at 1733279686477 (+4 ms)Running coprocessor post-close hooks at 1733279686482 (+5 ms)Closed at 1733279686483 (+1 ms) 2024-12-04T02:34:46,483 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733279679313.66b379fc7c2902adaf1f117fe455c1d2. 2024-12-04T02:34:46,483 DEBUG [RS_CLOSE_META-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T02:34:46,483 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c843721ad153bda320efe8a6bbfce0a4, disabling compactions & flushes 2024-12-04T02:34:46,483 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4. 2024-12-04T02:34:46,483 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4. 2024-12-04T02:34:46,483 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4. after waiting 0 ms 2024-12-04T02:34:46,483 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4. 2024-12-04T02:34:46,483 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c843721ad153bda320efe8a6bbfce0a4 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-04T02:34:46,499 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/c843721ad153bda320efe8a6bbfce0a4/.tmp/testFamily/296bd70fda0d48fdab40a27425b5a277 is 61, key is row-0/testFamily:testQualifier/1733279666204/Put/seqid=0 2024-12-04T02:34:46,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741861_1037 (size=5049) 2024-12-04T02:34:46,664 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on 05e988be3b28b57da7eaaf731591b3b5, 20d9ebc06c69a540e5cbfa93071b4415, 358638b58760ba89e229465286e54e93, 53b175aca7affc4c1f7e036710ba4734, a921ec0371ebd57e4fc59d2d75b9cee9, be4fba22bce0f0993f9e6bdd23cbac06, c843721ad153bda320efe8a6bbfce0a4 2024-12-04T02:34:46,865 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on 05e988be3b28b57da7eaaf731591b3b5, 20d9ebc06c69a540e5cbfa93071b4415, 358638b58760ba89e229465286e54e93, 53b175aca7affc4c1f7e036710ba4734, a921ec0371ebd57e4fc59d2d75b9cee9, be4fba22bce0f0993f9e6bdd23cbac06, c843721ad153bda320efe8a6bbfce0a4 2024-12-04T02:34:46,905 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/c843721ad153bda320efe8a6bbfce0a4/.tmp/testFamily/296bd70fda0d48fdab40a27425b5a277 2024-12-04T02:34:46,913 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/c843721ad153bda320efe8a6bbfce0a4/.tmp/testFamily/296bd70fda0d48fdab40a27425b5a277 as hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/c843721ad153bda320efe8a6bbfce0a4/testFamily/296bd70fda0d48fdab40a27425b5a277 2024-12-04T02:34:46,921 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/c843721ad153bda320efe8a6bbfce0a4/testFamily/296bd70fda0d48fdab40a27425b5a277, entries=4, sequenceid=8, filesize=4.9 K 2024-12-04T02:34:46,922 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c843721ad153bda320efe8a6bbfce0a4 in 439ms, sequenceid=8, compaction requested=false 2024-12-04T02:34:46,928 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/c843721ad153bda320efe8a6bbfce0a4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-04T02:34:46,929 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4. 2024-12-04T02:34:46,929 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c843721ad153bda320efe8a6bbfce0a4: Waiting for close lock at 1733279686483Running coprocessor pre-close hooks at 1733279686483Disabling compacts and flushes for region at 1733279686483Disabling writes for close at 1733279686483Obtaining lock to block concurrent updates at 1733279686483Preparing flush snapshotting stores in c843721ad153bda320efe8a6bbfce0a4 at 1733279686483Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733279686484 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4. at 1733279686484Flushing c843721ad153bda320efe8a6bbfce0a4/testFamily: creating writer at 1733279686484Flushing c843721ad153bda320efe8a6bbfce0a4/testFamily: appending metadata at 1733279686499 (+15 ms)Flushing c843721ad153bda320efe8a6bbfce0a4/testFamily: closing flushed file at 1733279686499Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@716cd0d: reopening flushed file at 1733279686912 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c843721ad153bda320efe8a6bbfce0a4 in 439ms, sequenceid=8, compaction requested=false at 1733279686922 (+10 ms)Writing region close event to WAL at 1733279686923 (+1 ms)Running coprocessor post-close hooks at 1733279686929 (+6 ms)Closed at 1733279686929 2024-12-04T02:34:46,929 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733279665179.c843721ad153bda320efe8a6bbfce0a4. 2024-12-04T02:34:46,929 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 358638b58760ba89e229465286e54e93, disabling compactions & flushes 2024-12-04T02:34:46,929 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93. 2024-12-04T02:34:46,929 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93. 2024-12-04T02:34:46,930 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93. after waiting 0 ms 2024-12-04T02:34:46,930 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93. 2024-12-04T02:34:46,930 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 358638b58760ba89e229465286e54e93 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-04T02:34:46,947 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testRetryOutOfOrderScannerNextExceptionAsync/358638b58760ba89e229465286e54e93/.tmp/testFamily/0726eaf0ad1f4940af8c485bfe049db8 is 61, key is row-0/testFamily:testQualifier/1733279627506/Put/seqid=0 2024-12-04T02:34:46,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741862_1038 (size=5049) 2024-12-04T02:34:47,065 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on 05e988be3b28b57da7eaaf731591b3b5, 20d9ebc06c69a540e5cbfa93071b4415, 358638b58760ba89e229465286e54e93, 53b175aca7affc4c1f7e036710ba4734, a921ec0371ebd57e4fc59d2d75b9cee9, be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:34:47,266 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on 05e988be3b28b57da7eaaf731591b3b5, 20d9ebc06c69a540e5cbfa93071b4415, 358638b58760ba89e229465286e54e93, 53b175aca7affc4c1f7e036710ba4734, a921ec0371ebd57e4fc59d2d75b9cee9, be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:34:47,354 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testRetryOutOfOrderScannerNextExceptionAsync/358638b58760ba89e229465286e54e93/.tmp/testFamily/0726eaf0ad1f4940af8c485bfe049db8 2024-12-04T02:34:47,370 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testRetryOutOfOrderScannerNextExceptionAsync/358638b58760ba89e229465286e54e93/.tmp/testFamily/0726eaf0ad1f4940af8c485bfe049db8 as hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testRetryOutOfOrderScannerNextExceptionAsync/358638b58760ba89e229465286e54e93/testFamily/0726eaf0ad1f4940af8c485bfe049db8 2024-12-04T02:34:47,381 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testRetryOutOfOrderScannerNextExceptionAsync/358638b58760ba89e229465286e54e93/testFamily/0726eaf0ad1f4940af8c485bfe049db8, entries=4, sequenceid=8, filesize=4.9 K 2024-12-04T02:34:47,383 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 358638b58760ba89e229465286e54e93 in 453ms, sequenceid=8, compaction requested=false 2024-12-04T02:34:47,389 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testRetryOutOfOrderScannerNextExceptionAsync/358638b58760ba89e229465286e54e93/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-04T02:34:47,391 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93. 2024-12-04T02:34:47,391 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 358638b58760ba89e229465286e54e93: Waiting for close lock at 1733279686929Running coprocessor pre-close hooks at 1733279686929Disabling compacts and flushes for region at 1733279686929Disabling writes for close at 1733279686930 (+1 ms)Obtaining lock to block concurrent updates at 1733279686930Preparing flush snapshotting stores in 358638b58760ba89e229465286e54e93 at 1733279686930Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733279686930Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93. at 1733279686931 (+1 ms)Flushing 358638b58760ba89e229465286e54e93/testFamily: creating writer at 1733279686931Flushing 358638b58760ba89e229465286e54e93/testFamily: appending metadata at 1733279686946 (+15 ms)Flushing 358638b58760ba89e229465286e54e93/testFamily: closing flushed file at 1733279686946Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f0f639d: reopening flushed file at 1733279687368 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 358638b58760ba89e229465286e54e93 in 453ms, sequenceid=8, compaction requested=false at 1733279687383 (+15 ms)Writing region close event to WAL at 1733279687384 (+1 ms)Running coprocessor post-close hooks at 1733279687390 (+6 ms)Closed at 1733279687390 2024-12-04T02:34:47,391 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733279624431.358638b58760ba89e229465286e54e93. 2024-12-04T02:34:47,391 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a921ec0371ebd57e4fc59d2d75b9cee9, disabling compactions & flushes 2024-12-04T02:34:47,391 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9. 2024-12-04T02:34:47,391 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9. 2024-12-04T02:34:47,391 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9. after waiting 0 ms 2024-12-04T02:34:47,391 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9. 2024-12-04T02:34:47,391 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a921ec0371ebd57e4fc59d2d75b9cee9 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-04T02:34:47,413 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnOpenScanner/a921ec0371ebd57e4fc59d2d75b9cee9/.tmp/testFamily/24690bd19a1548b491ec0dc3eb5707ff is 61, key is row-0/testFamily:testQualifier/1733279651984/Put/seqid=0 2024-12-04T02:34:47,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741863_1039 (size=5049) 2024-12-04T02:34:47,466 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1321): Waiting on 5 regions to close 2024-12-04T02:34:47,466 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1325): Online Regions={a921ec0371ebd57e4fc59d2d75b9cee9=testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9., 53b175aca7affc4c1f7e036710ba4734=testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734., 05e988be3b28b57da7eaaf731591b3b5=testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5., 20d9ebc06c69a540e5cbfa93071b4415=testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415., be4fba22bce0f0993f9e6bdd23cbac06=hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06.} 2024-12-04T02:34:47,466 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on 05e988be3b28b57da7eaaf731591b3b5, 20d9ebc06c69a540e5cbfa93071b4415, 53b175aca7affc4c1f7e036710ba4734, a921ec0371ebd57e4fc59d2d75b9cee9, be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:34:47,666 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on 05e988be3b28b57da7eaaf731591b3b5, 20d9ebc06c69a540e5cbfa93071b4415, 53b175aca7affc4c1f7e036710ba4734, a921ec0371ebd57e4fc59d2d75b9cee9, be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:34:47,824 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnOpenScanner/a921ec0371ebd57e4fc59d2d75b9cee9/.tmp/testFamily/24690bd19a1548b491ec0dc3eb5707ff 2024-12-04T02:34:47,834 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnOpenScanner/a921ec0371ebd57e4fc59d2d75b9cee9/.tmp/testFamily/24690bd19a1548b491ec0dc3eb5707ff as hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnOpenScanner/a921ec0371ebd57e4fc59d2d75b9cee9/testFamily/24690bd19a1548b491ec0dc3eb5707ff 2024-12-04T02:34:47,845 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnOpenScanner/a921ec0371ebd57e4fc59d2d75b9cee9/testFamily/24690bd19a1548b491ec0dc3eb5707ff, entries=4, sequenceid=8, filesize=4.9 K 2024-12-04T02:34:47,846 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a921ec0371ebd57e4fc59d2d75b9cee9 in 455ms, sequenceid=8, compaction requested=false 2024-12-04T02:34:47,851 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnOpenScanner/a921ec0371ebd57e4fc59d2d75b9cee9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-04T02:34:47,852 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9. 2024-12-04T02:34:47,852 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a921ec0371ebd57e4fc59d2d75b9cee9: Waiting for close lock at 1733279687391Running coprocessor pre-close hooks at 1733279687391Disabling compacts and flushes for region at 1733279687391Disabling writes for close at 1733279687391Obtaining lock to block concurrent updates at 1733279687391Preparing flush snapshotting stores in a921ec0371ebd57e4fc59d2d75b9cee9 at 1733279687391Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733279687392 (+1 ms)Flushing stores of testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9. at 1733279687392Flushing a921ec0371ebd57e4fc59d2d75b9cee9/testFamily: creating writer at 1733279687393 (+1 ms)Flushing a921ec0371ebd57e4fc59d2d75b9cee9/testFamily: appending metadata at 1733279687412 (+19 ms)Flushing a921ec0371ebd57e4fc59d2d75b9cee9/testFamily: closing flushed file at 1733279687412Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64a54147: reopening flushed file at 1733279687833 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a921ec0371ebd57e4fc59d2d75b9cee9 in 455ms, sequenceid=8, compaction requested=false at 1733279687846 (+13 ms)Writing region close event to WAL at 1733279687847 (+1 ms)Running coprocessor post-close hooks at 1733279687852 (+5 ms)Closed at 1733279687852 2024-12-04T02:34:47,853 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1733279648926.a921ec0371ebd57e4fc59d2d75b9cee9. 2024-12-04T02:34:47,853 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 53b175aca7affc4c1f7e036710ba4734, disabling compactions & flushes 2024-12-04T02:34:47,853 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734. 2024-12-04T02:34:47,853 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734. 2024-12-04T02:34:47,853 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734. after waiting 0 ms 2024-12-04T02:34:47,853 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734. 2024-12-04T02:34:47,853 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 53b175aca7affc4c1f7e036710ba4734 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-04T02:34:47,867 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on 05e988be3b28b57da7eaaf731591b3b5, 20d9ebc06c69a540e5cbfa93071b4415, 53b175aca7affc4c1f7e036710ba4734, be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:34:47,868 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnNextAsync/53b175aca7affc4c1f7e036710ba4734/.tmp/testFamily/4bfb477a361f44f5ba36c309d786556d is 61, key is row-0/testFamily:testQualifier/1733279656061/Put/seqid=0 2024-12-04T02:34:47,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741864_1040 (size=5049) 2024-12-04T02:34:48,067 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on 05e988be3b28b57da7eaaf731591b3b5, 20d9ebc06c69a540e5cbfa93071b4415, 53b175aca7affc4c1f7e036710ba4734, be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:34:48,267 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on 05e988be3b28b57da7eaaf731591b3b5, 20d9ebc06c69a540e5cbfa93071b4415, 53b175aca7affc4c1f7e036710ba4734, be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:34:48,273 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnNextAsync/53b175aca7affc4c1f7e036710ba4734/.tmp/testFamily/4bfb477a361f44f5ba36c309d786556d 2024-12-04T02:34:48,285 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnNextAsync/53b175aca7affc4c1f7e036710ba4734/.tmp/testFamily/4bfb477a361f44f5ba36c309d786556d as hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnNextAsync/53b175aca7affc4c1f7e036710ba4734/testFamily/4bfb477a361f44f5ba36c309d786556d 2024-12-04T02:34:48,293 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnNextAsync/53b175aca7affc4c1f7e036710ba4734/testFamily/4bfb477a361f44f5ba36c309d786556d, entries=4, sequenceid=8, filesize=4.9 K 2024-12-04T02:34:48,294 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 53b175aca7affc4c1f7e036710ba4734 in 441ms, sequenceid=8, compaction requested=false 2024-12-04T02:34:48,299 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnNextAsync/53b175aca7affc4c1f7e036710ba4734/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-04T02:34:48,300 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734. 2024-12-04T02:34:48,300 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 53b175aca7affc4c1f7e036710ba4734: Waiting for close lock at 1733279687853Running coprocessor pre-close hooks at 1733279687853Disabling compacts and flushes for region at 1733279687853Disabling writes for close at 1733279687853Obtaining lock to block concurrent updates at 1733279687853Preparing flush snapshotting stores in 53b175aca7affc4c1f7e036710ba4734 at 1733279687853Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733279687853Flushing stores of testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734. at 1733279687854 (+1 ms)Flushing 53b175aca7affc4c1f7e036710ba4734/testFamily: creating writer at 1733279687854Flushing 53b175aca7affc4c1f7e036710ba4734/testFamily: appending metadata at 1733279687867 (+13 ms)Flushing 53b175aca7affc4c1f7e036710ba4734/testFamily: closing flushed file at 1733279687867Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16d52913: reopening flushed file at 1733279688283 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 53b175aca7affc4c1f7e036710ba4734 in 441ms, sequenceid=8, compaction requested=false at 1733279688294 (+11 ms)Writing region close event to WAL at 1733279688295 (+1 ms)Running coprocessor post-close hooks at 1733279688299 (+4 ms)Closed at 1733279688300 (+1 ms) 2024-12-04T02:34:48,300 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1733279655028.53b175aca7affc4c1f7e036710ba4734. 2024-12-04T02:34:48,300 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 05e988be3b28b57da7eaaf731591b3b5, disabling compactions & flushes 2024-12-04T02:34:48,300 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5. 2024-12-04T02:34:48,300 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5. 2024-12-04T02:34:48,300 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5. after waiting 0 ms 2024-12-04T02:34:48,300 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5. 2024-12-04T02:34:48,301 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 05e988be3b28b57da7eaaf731591b3b5 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-04T02:34:48,323 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnNext/05e988be3b28b57da7eaaf731591b3b5/.tmp/testFamily/c25a035969d14d3c8e4e80111d2da055 is 61, key is row-0/testFamily:testQualifier/1733279662141/Put/seqid=0 2024-12-04T02:34:48,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741865_1041 (size=5049) 2024-12-04T02:34:48,468 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-12-04T02:34:48,468 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1325): Online Regions={05e988be3b28b57da7eaaf731591b3b5=testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5., 20d9ebc06c69a540e5cbfa93071b4415=testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415., be4fba22bce0f0993f9e6bdd23cbac06=hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06.} 2024-12-04T02:34:48,468 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on 05e988be3b28b57da7eaaf731591b3b5, 20d9ebc06c69a540e5cbfa93071b4415, be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:34:48,669 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on 05e988be3b28b57da7eaaf731591b3b5, 20d9ebc06c69a540e5cbfa93071b4415, be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:34:48,729 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnNext/05e988be3b28b57da7eaaf731591b3b5/.tmp/testFamily/c25a035969d14d3c8e4e80111d2da055 2024-12-04T02:34:48,741 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnNext/05e988be3b28b57da7eaaf731591b3b5/.tmp/testFamily/c25a035969d14d3c8e4e80111d2da055 as hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnNext/05e988be3b28b57da7eaaf731591b3b5/testFamily/c25a035969d14d3c8e4e80111d2da055 2024-12-04T02:34:48,748 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnNext/05e988be3b28b57da7eaaf731591b3b5/testFamily/c25a035969d14d3c8e4e80111d2da055, entries=4, sequenceid=8, filesize=4.9 K 2024-12-04T02:34:48,749 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 05e988be3b28b57da7eaaf731591b3b5 in 448ms, sequenceid=8, compaction requested=false 2024-12-04T02:34:48,754 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testNormalScanTimeoutOnNext/05e988be3b28b57da7eaaf731591b3b5/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-04T02:34:48,754 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5. 2024-12-04T02:34:48,754 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 05e988be3b28b57da7eaaf731591b3b5: Waiting for close lock at 1733279688300Running coprocessor pre-close hooks at 1733279688300Disabling compacts and flushes for region at 1733279688300Disabling writes for close at 1733279688300Obtaining lock to block concurrent updates at 1733279688301 (+1 ms)Preparing flush snapshotting stores in 05e988be3b28b57da7eaaf731591b3b5 at 1733279688301Finished memstore snapshotting testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733279688301Flushing stores of testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5. at 1733279688302 (+1 ms)Flushing 05e988be3b28b57da7eaaf731591b3b5/testFamily: creating writer at 1733279688302Flushing 05e988be3b28b57da7eaaf731591b3b5/testFamily: appending metadata at 1733279688323 (+21 ms)Flushing 05e988be3b28b57da7eaaf731591b3b5/testFamily: closing flushed file at 1733279688323Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7304d004: reopening flushed file at 1733279688740 (+417 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 05e988be3b28b57da7eaaf731591b3b5 in 448ms, sequenceid=8, compaction requested=false at 1733279688749 (+9 ms)Writing region close event to WAL at 1733279688750 (+1 ms)Running coprocessor post-close hooks at 1733279688754 (+4 ms)Closed at 1733279688754 2024-12-04T02:34:48,755 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1733279659109.05e988be3b28b57da7eaaf731591b3b5. 2024-12-04T02:34:48,755 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 20d9ebc06c69a540e5cbfa93071b4415, disabling compactions & flushes 2024-12-04T02:34:48,755 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415. 2024-12-04T02:34:48,755 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415. 2024-12-04T02:34:48,755 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415. after waiting 0 ms 2024-12-04T02:34:48,755 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415. 2024-12-04T02:34:48,755 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 20d9ebc06c69a540e5cbfa93071b4415 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-04T02:34:48,772 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testRetryOutOfOrderScannerNextException/20d9ebc06c69a540e5cbfa93071b4415/.tmp/testFamily/abb1192e38f94f56b77d7c02f24beffb is 61, key is row-0/testFamily:testQualifier/1733279648875/Put/seqid=0 2024-12-04T02:34:48,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741866_1042 (size=5049) 2024-12-04T02:34:48,869 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on 20d9ebc06c69a540e5cbfa93071b4415, be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:34:49,069 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on 20d9ebc06c69a540e5cbfa93071b4415, be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:34:49,179 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testRetryOutOfOrderScannerNextException/20d9ebc06c69a540e5cbfa93071b4415/.tmp/testFamily/abb1192e38f94f56b77d7c02f24beffb 2024-12-04T02:34:49,190 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testRetryOutOfOrderScannerNextException/20d9ebc06c69a540e5cbfa93071b4415/.tmp/testFamily/abb1192e38f94f56b77d7c02f24beffb as hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testRetryOutOfOrderScannerNextException/20d9ebc06c69a540e5cbfa93071b4415/testFamily/abb1192e38f94f56b77d7c02f24beffb 2024-12-04T02:34:49,197 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testRetryOutOfOrderScannerNextException/20d9ebc06c69a540e5cbfa93071b4415/testFamily/abb1192e38f94f56b77d7c02f24beffb, entries=4, sequenceid=8, filesize=4.9 K 2024-12-04T02:34:49,198 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 20d9ebc06c69a540e5cbfa93071b4415 in 443ms, sequenceid=8, compaction requested=false 2024-12-04T02:34:49,203 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/default/testRetryOutOfOrderScannerNextException/20d9ebc06c69a540e5cbfa93071b4415/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-04T02:34:49,203 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415. 2024-12-04T02:34:49,203 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 20d9ebc06c69a540e5cbfa93071b4415: Waiting for close lock at 1733279688755Running coprocessor pre-close hooks at 1733279688755Disabling compacts and flushes for region at 1733279688755Disabling writes for close at 1733279688755Obtaining lock to block concurrent updates at 1733279688755Preparing flush snapshotting stores in 20d9ebc06c69a540e5cbfa93071b4415 at 1733279688755Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733279688755Flushing stores of testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415. at 1733279688756 (+1 ms)Flushing 20d9ebc06c69a540e5cbfa93071b4415/testFamily: creating writer at 1733279688756Flushing 20d9ebc06c69a540e5cbfa93071b4415/testFamily: appending metadata at 1733279688772 (+16 ms)Flushing 20d9ebc06c69a540e5cbfa93071b4415/testFamily: closing flushed file at 1733279688772Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@677f7080: reopening flushed file at 1733279689189 (+417 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 20d9ebc06c69a540e5cbfa93071b4415 in 443ms, sequenceid=8, compaction requested=false at 1733279689198 (+9 ms)Writing region close event to WAL at 1733279689199 (+1 ms)Running coprocessor post-close hooks at 1733279689203 (+4 ms)Closed at 1733279689203 2024-12-04T02:34:49,204 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1733279645827.20d9ebc06c69a540e5cbfa93071b4415. 2024-12-04T02:34:49,204 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing be4fba22bce0f0993f9e6bdd23cbac06, disabling compactions & flushes 2024-12-04T02:34:49,204 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06. 2024-12-04T02:34:49,204 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06. 2024-12-04T02:34:49,204 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06. after waiting 0 ms 2024-12-04T02:34:49,204 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06. 2024-12-04T02:34:49,204 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing be4fba22bce0f0993f9e6bdd23cbac06 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-04T02:34:49,218 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnNextAsync/be4fba22bce0f0993f9e6bdd23cbac06/.tmp/testFamily/953cca3209ea463087829d99f72c6b5c is 61, key is row-0/testFamily:testQualifier/1733279639763/Put/seqid=0 2024-12-04T02:34:49,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741867_1043 (size=5049) 2024-12-04T02:34:49,270 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:34:49,470 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T02:34:49,470 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1325): Online Regions={be4fba22bce0f0993f9e6bdd23cbac06=hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06.} 2024-12-04T02:34:49,470 DEBUG [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1351): Waiting on be4fba22bce0f0993f9e6bdd23cbac06 2024-12-04T02:34:49,624 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnNextAsync/be4fba22bce0f0993f9e6bdd23cbac06/.tmp/testFamily/953cca3209ea463087829d99f72c6b5c 2024-12-04T02:34:49,633 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnNextAsync/be4fba22bce0f0993f9e6bdd23cbac06/.tmp/testFamily/953cca3209ea463087829d99f72c6b5c as hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnNextAsync/be4fba22bce0f0993f9e6bdd23cbac06/testFamily/953cca3209ea463087829d99f72c6b5c 2024-12-04T02:34:49,640 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnNextAsync/be4fba22bce0f0993f9e6bdd23cbac06/testFamily/953cca3209ea463087829d99f72c6b5c, entries=4, sequenceid=8, filesize=4.9 K 2024-12-04T02:34:49,641 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for be4fba22bce0f0993f9e6bdd23cbac06 in 437ms, sequenceid=8, compaction requested=false 2024-12-04T02:34:49,646 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/data/hbase/testMetaScanTimeoutOnNextAsync/be4fba22bce0f0993f9e6bdd23cbac06/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-04T02:34:49,646 INFO [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06. 2024-12-04T02:34:49,647 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for be4fba22bce0f0993f9e6bdd23cbac06: Waiting for close lock at 1733279689204Running coprocessor pre-close hooks at 1733279689204Disabling compacts and flushes for region at 1733279689204Disabling writes for close at 1733279689204Obtaining lock to block concurrent updates at 1733279689204Preparing flush snapshotting stores in be4fba22bce0f0993f9e6bdd23cbac06 at 1733279689204Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733279689204Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06. at 1733279689205 (+1 ms)Flushing be4fba22bce0f0993f9e6bdd23cbac06/testFamily: creating writer at 1733279689205Flushing be4fba22bce0f0993f9e6bdd23cbac06/testFamily: appending metadata at 1733279689218 (+13 ms)Flushing be4fba22bce0f0993f9e6bdd23cbac06/testFamily: closing flushed file at 1733279689218Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7196face: reopening flushed file at 1733279689632 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for be4fba22bce0f0993f9e6bdd23cbac06 in 437ms, sequenceid=8, compaction requested=false at 1733279689641 (+9 ms)Writing region close event to WAL at 1733279689642 (+1 ms)Running coprocessor post-close hooks at 1733279689646 (+4 ms)Closed at 1733279689646 2024-12-04T02:34:49,647 DEBUG [RS_CLOSE_REGION-regionserver/40249addbad9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733279636710.be4fba22bce0f0993f9e6bdd23cbac06. 2024-12-04T02:34:49,670 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(976): stopping server 40249addbad9,43941,1733279590307; all regions closed. 2024-12-04T02:34:49,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741834_1010 (size=19479) 2024-12-04T02:34:49,686 DEBUG [RS:0;40249addbad9:43941 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/oldWALs 2024-12-04T02:34:49,686 INFO [RS:0;40249addbad9:43941 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 40249addbad9%2C43941%2C1733279590307.meta:.meta(num 1733279623829) 2024-12-04T02:34:49,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741833_1009 (size=20031) 2024-12-04T02:34:49,692 DEBUG [RS:0;40249addbad9:43941 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/oldWALs 2024-12-04T02:34:49,692 INFO [RS:0;40249addbad9:43941 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 40249addbad9%2C43941%2C1733279590307:(num 1733279623398) 2024-12-04T02:34:49,692 DEBUG [RS:0;40249addbad9:43941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T02:34:49,693 INFO [RS:0;40249addbad9:43941 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T02:34:49,693 INFO [RS:0;40249addbad9:43941 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T02:34:49,693 INFO [RS:0;40249addbad9:43941 {}] hbase.ChoreService(370): Chore service for: regionserver/40249addbad9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T02:34:49,693 INFO [RS:0;40249addbad9:43941 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T02:34:49,693 INFO [regionserver/40249addbad9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T02:34:49,695 INFO [RS:0;40249addbad9:43941 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43941 2024-12-04T02:34:49,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T02:34:49,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43941-0x1019e2f9c250001, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/40249addbad9,43941,1733279590307 2024-12-04T02:34:49,708 INFO [RS:0;40249addbad9:43941 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T02:34:49,719 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [40249addbad9,43941,1733279590307] 2024-12-04T02:34:49,729 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/40249addbad9,43941,1733279590307 already deleted, retry=false 2024-12-04T02:34:49,729 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 40249addbad9,43941,1733279590307 expired; onlineServers=0 2024-12-04T02:34:49,729 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '40249addbad9,43601,1733279589466' ***** 2024-12-04T02:34:49,729 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T02:34:49,729 INFO [M:0;40249addbad9:43601 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T02:34:49,730 INFO [M:0;40249addbad9:43601 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T02:34:49,730 DEBUG [M:0;40249addbad9:43601 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T02:34:49,730 DEBUG [M:0;40249addbad9:43601 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T02:34:49,730 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T02:34:49,730 DEBUG [master/40249addbad9:0:becomeActiveMaster-HFileCleaner.large.0-1733279593199 {}] cleaner.HFileCleaner(306): Exit Thread[master/40249addbad9:0:becomeActiveMaster-HFileCleaner.large.0-1733279593199,5,FailOnTimeoutGroup] 2024-12-04T02:34:49,730 DEBUG [master/40249addbad9:0:becomeActiveMaster-HFileCleaner.small.0-1733279593200 {}] cleaner.HFileCleaner(306): Exit Thread[master/40249addbad9:0:becomeActiveMaster-HFileCleaner.small.0-1733279593200,5,FailOnTimeoutGroup] 2024-12-04T02:34:49,730 INFO [M:0;40249addbad9:43601 {}] hbase.ChoreService(370): Chore service for: master/40249addbad9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T02:34:49,731 INFO [M:0;40249addbad9:43601 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T02:34:49,731 DEBUG [M:0;40249addbad9:43601 {}] master.HMaster(1795): Stopping service threads 2024-12-04T02:34:49,731 INFO [M:0;40249addbad9:43601 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T02:34:49,731 INFO [M:0;40249addbad9:43601 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T02:34:49,732 INFO [M:0;40249addbad9:43601 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T02:34:49,732 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T02:34:49,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T02:34:49,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T02:34:49,740 DEBUG [M:0;40249addbad9:43601 {}] zookeeper.ZKUtil(347): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T02:34:49,740 WARN [M:0;40249addbad9:43601 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T02:34:49,741 INFO [M:0;40249addbad9:43601 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/.lastflushedseqids 2024-12-04T02:34:49,751 INFO [regionserver/40249addbad9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T02:34:49,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741868_1044 (size=712) 2024-12-04T02:34:49,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43941-0x1019e2f9c250001, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T02:34:49,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43941-0x1019e2f9c250001, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T02:34:49,820 INFO [RS:0;40249addbad9:43941 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T02:34:49,820 INFO [RS:0;40249addbad9:43941 {}] regionserver.HRegionServer(1031): Exiting; stopping=40249addbad9,43941,1733279590307; zookeeper connection closed. 2024-12-04T02:34:49,821 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@72dab5a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@72dab5a 2024-12-04T02:34:49,821 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-04T02:34:49,900 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-04T02:34:49,900 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-04T02:34:49,901 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T02:34:49,901 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T02:34:49,901 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-04T02:34:49,901 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-04T02:34:49,901 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-04T02:34:49,902 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-04T02:34:49,902 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-04T02:34:49,902 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-04T02:34:49,902 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-04T02:34:50,157 INFO [M:0;40249addbad9:43601 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T02:34:50,157 INFO [M:0;40249addbad9:43601 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T02:34:50,157 DEBUG [M:0;40249addbad9:43601 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T02:34:50,158 INFO [M:0;40249addbad9:43601 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T02:34:50,158 DEBUG [M:0;40249addbad9:43601 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T02:34:50,158 DEBUG [M:0;40249addbad9:43601 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T02:34:50,158 DEBUG [M:0;40249addbad9:43601 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T02:34:50,158 INFO [M:0;40249addbad9:43601 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.15 KB heapSize=191.20 KB 2024-12-04T02:34:50,173 DEBUG [M:0;40249addbad9:43601 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5d59f57f8dbf4e9ca11abf85e2da8cb7 is 82, key is hbase:meta,,1/info:regioninfo/1733279623941/Put/seqid=0 2024-12-04T02:34:50,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741869_1045 (size=5672) 2024-12-04T02:34:50,579 INFO [M:0;40249addbad9:43601 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5d59f57f8dbf4e9ca11abf85e2da8cb7 2024-12-04T02:34:50,615 DEBUG [M:0;40249addbad9:43601 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/006919abe72349208e4a989d3ffa127f is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733279625702/Put/seqid=0 2024-12-04T02:34:50,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741870_1046 (size=14076) 2024-12-04T02:34:51,021 INFO [M:0;40249addbad9:43601 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.59 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/006919abe72349208e4a989d3ffa127f 2024-12-04T02:34:51,029 INFO [M:0;40249addbad9:43601 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 006919abe72349208e4a989d3ffa127f 2024-12-04T02:34:51,045 DEBUG [M:0;40249addbad9:43601 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/43afd4c131bf44e7bac1d54e000259af is 69, key is 40249addbad9,43941,1733279590307/rs:state/1733279593274/Put/seqid=0 2024-12-04T02:34:51,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741871_1047 (size=5156) 2024-12-04T02:34:51,451 INFO [M:0;40249addbad9:43601 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/43afd4c131bf44e7bac1d54e000259af 2024-12-04T02:34:51,462 DEBUG [M:0;40249addbad9:43601 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5d59f57f8dbf4e9ca11abf85e2da8cb7 as hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5d59f57f8dbf4e9ca11abf85e2da8cb7 2024-12-04T02:34:51,471 INFO [M:0;40249addbad9:43601 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5d59f57f8dbf4e9ca11abf85e2da8cb7, entries=8, sequenceid=319, filesize=5.5 K 2024-12-04T02:34:51,473 DEBUG [M:0;40249addbad9:43601 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/006919abe72349208e4a989d3ffa127f as hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/006919abe72349208e4a989d3ffa127f 2024-12-04T02:34:51,478 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T02:34:51,480 INFO [M:0;40249addbad9:43601 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 006919abe72349208e4a989d3ffa127f 2024-12-04T02:34:51,480 INFO [M:0;40249addbad9:43601 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/006919abe72349208e4a989d3ffa127f, entries=33, sequenceid=319, filesize=13.7 K 2024-12-04T02:34:51,482 DEBUG [M:0;40249addbad9:43601 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/43afd4c131bf44e7bac1d54e000259af as hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/43afd4c131bf44e7bac1d54e000259af 2024-12-04T02:34:51,489 INFO [M:0;40249addbad9:43601 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34037/user/jenkins/test-data/7093db6d-e677-4c0c-00d6-b6ac38ac0bc7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/43afd4c131bf44e7bac1d54e000259af, entries=1, sequenceid=319, filesize=5.0 K 2024-12-04T02:34:51,490 INFO [M:0;40249addbad9:43601 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.15 KB/167063, heapSize ~190.91 KB/195488, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1332ms, sequenceid=319, compaction requested=false 2024-12-04T02:34:51,492 INFO [M:0;40249addbad9:43601 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T02:34:51,492 DEBUG [M:0;40249addbad9:43601 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733279690157Disabling compacts and flushes for region at 1733279690157Disabling writes for close at 1733279690158 (+1 ms)Obtaining lock to block concurrent updates at 1733279690158Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733279690158Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=167063, getHeapSize=195728, getOffHeapSize=0, getCellsCount=366 at 1733279690158Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733279690159 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733279690159Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733279690172 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733279690172Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733279690593 (+421 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733279690615 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733279690615Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733279691029 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733279691045 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733279691045Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3389cbea: reopening flushed file at 1733279691460 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49b2c7: reopening flushed file at 1733279691471 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1acd8761: reopening flushed file at 1733279691481 (+10 ms)Finished flush of dataSize ~163.15 KB/167063, heapSize ~190.91 KB/195488, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1332ms, sequenceid=319, compaction requested=false at 1733279691490 (+9 ms)Writing region close event to WAL at 1733279691492 (+2 ms)Closed at 1733279691492 2024-12-04T02:34:51,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33875 is added to blk_1073741830_1006 (size=188405) 2024-12-04T02:34:51,496 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T02:34:51,496 INFO [M:0;40249addbad9:43601 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T02:34:51,496 INFO [M:0;40249addbad9:43601 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43601 2024-12-04T02:34:51,497 INFO [M:0;40249addbad9:43601 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T02:34:51,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T02:34:51,660 INFO [M:0;40249addbad9:43601 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T02:34:51,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43601-0x1019e2f9c250000, quorum=127.0.0.1:53960, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T02:34:51,665 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5bc105ee{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T02:34:51,667 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@360caa46{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T02:34:51,667 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T02:34:51,667 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@30fd9473{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T02:34:51,668 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f7444f6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/hadoop.log.dir/,STOPPED} 2024-12-04T02:34:51,670 WARN [BP-2088324271-172.17.0.3-1733279584749 heartbeating to localhost/127.0.0.1:34037 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T02:34:51,670 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T02:34:51,670 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T02:34:51,670 WARN [BP-2088324271-172.17.0.3-1733279584749 heartbeating to localhost/127.0.0.1:34037 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2088324271-172.17.0.3-1733279584749 (Datanode Uuid 88c4e69d-6a95-4d0f-b6bb-d93f38f96390) service to localhost/127.0.0.1:34037 2024-12-04T02:34:51,671 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/cluster_94449acd-a2fa-e53b-a6f8-948a65223f61/data/data1/current/BP-2088324271-172.17.0.3-1733279584749 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T02:34:51,672 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/cluster_94449acd-a2fa-e53b-a6f8-948a65223f61/data/data2/current/BP-2088324271-172.17.0.3-1733279584749 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T02:34:51,672 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T02:34:51,683 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3ce6ffde{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T02:34:51,684 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5c90255f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T02:34:51,684 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T02:34:51,684 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3959e81b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T02:34:51,684 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3bcd68eb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f381c65e-a452-0f21-31c7-abeb2785ed2e/hadoop.log.dir/,STOPPED} 2024-12-04T02:34:51,694 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T02:34:51,723 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down