2024-12-04 19:24:27,444 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-04 19:24:27,454 main DEBUG Took 0.007882 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-04 19:24:27,454 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-04 19:24:27,454 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-04 19:24:27,455 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-04 19:24:27,456 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:24:27,462 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-04 19:24:27,472 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:24:27,474 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:24:27,474 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:24:27,475 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:24:27,475 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:24:27,475 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:24:27,476 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:24:27,476 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:24:27,477 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:24:27,477 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:24:27,478 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:24:27,478 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:24:27,478 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:24:27,478 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:24:27,479 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:24:27,479 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:24:27,479 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:24:27,480 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:24:27,480 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:24:27,480 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:24:27,480 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:24:27,481 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:24:27,481 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:24:27,481 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 19:24:27,482 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:24:27,482 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-04 19:24:27,483 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 19:24:27,484 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-04 19:24:27,486 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-04 19:24:27,486 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-04 19:24:27,487 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-04 19:24:27,487 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-04 19:24:27,494 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-04 19:24:27,497 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-04 19:24:27,498 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-04 19:24:27,498 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-04 19:24:27,499 main DEBUG createAppenders(={Console}) 2024-12-04 19:24:27,499 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-04 19:24:27,499 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-04 19:24:27,500 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-04 19:24:27,500 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-04 19:24:27,500 main DEBUG OutputStream closed 2024-12-04 19:24:27,501 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-04 19:24:27,501 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-04 19:24:27,501 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-04 19:24:27,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-04 19:24:27,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-04 19:24:27,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-04 19:24:27,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-04 19:24:27,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-04 19:24:27,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-04 19:24:27,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-04 19:24:27,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-04 19:24:27,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-04 19:24:27,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-04 19:24:27,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-04 19:24:27,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-04 19:24:27,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-04 19:24:27,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-04 19:24:27,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-04 19:24:27,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-04 19:24:27,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-04 19:24:27,571 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-04 19:24:27,573 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04 19:24:27,573 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-04 19:24:27,573 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-04 19:24:27,574 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-04T19:24:27,779 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1 2024-12-04 19:24:27,781 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-04 19:24:27,781 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04T19:24:27,789 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-04T19:24:27,795 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-12-04T19:24:27,815 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-04T19:24:27,861 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-04T19:24:27,862 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-04T19:24:27,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T19:24:27,885 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805, deleteOnExit=true 2024-12-04T19:24:27,886 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T19:24:27,887 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/test.cache.data in system properties and HBase conf 2024-12-04T19:24:27,887 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T19:24:27,887 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/hadoop.log.dir in system properties and HBase conf 2024-12-04T19:24:27,888 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T19:24:27,888 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T19:24:27,888 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T19:24:27,955 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-04T19:24:28,039 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T19:24:28,042 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T19:24:28,042 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T19:24:28,043 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T19:24:28,043 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T19:24:28,043 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T19:24:28,044 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T19:24:28,044 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T19:24:28,044 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T19:24:28,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T19:24:28,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/nfs.dump.dir in system properties and HBase conf 2024-12-04T19:24:28,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/java.io.tmpdir in system properties and HBase conf 2024-12-04T19:24:28,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T19:24:28,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T19:24:28,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T19:24:29,004 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-04T19:24:29,069 INFO [Time-limited test {}] log.Log(170): Logging initialized @2197ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-04T19:24:29,133 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T19:24:29,194 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T19:24:29,211 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T19:24:29,211 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T19:24:29,212 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T19:24:29,228 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T19:24:29,230 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/hadoop.log.dir/,AVAILABLE} 2024-12-04T19:24:29,231 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T19:24:29,383 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/java.io.tmpdir/jetty-localhost-35159-hadoop-hdfs-3_4_1-tests_jar-_-any-17611932825125854420/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T19:24:29,388 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:35159} 2024-12-04T19:24:29,388 INFO [Time-limited test {}] server.Server(415): Started @2516ms 2024-12-04T19:24:29,883 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T19:24:29,890 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T19:24:29,892 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T19:24:29,892 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T19:24:29,892 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T19:24:29,894 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/hadoop.log.dir/,AVAILABLE} 2024-12-04T19:24:29,894 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T19:24:29,988 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@501095cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/java.io.tmpdir/jetty-localhost-41393-hadoop-hdfs-3_4_1-tests_jar-_-any-14756879004458202618/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T19:24:29,989 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:41393} 2024-12-04T19:24:29,989 INFO [Time-limited test {}] server.Server(415): Started @3117ms 2024-12-04T19:24:30,032 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T19:24:30,133 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T19:24:30,139 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T19:24:30,140 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T19:24:30,141 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T19:24:30,141 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T19:24:30,142 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/hadoop.log.dir/,AVAILABLE} 2024-12-04T19:24:30,142 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T19:24:30,264 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64052af6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/java.io.tmpdir/jetty-localhost-33085-hadoop-hdfs-3_4_1-tests_jar-_-any-15876180316653329592/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T19:24:30,265 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:33085} 2024-12-04T19:24:30,265 INFO [Time-limited test {}] server.Server(415): Started @3393ms 2024-12-04T19:24:30,267 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T19:24:30,300 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T19:24:30,305 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T19:24:30,306 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T19:24:30,307 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T19:24:30,307 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T19:24:30,308 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/hadoop.log.dir/,AVAILABLE} 2024-12-04T19:24:30,308 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T19:24:30,420 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@420bc738{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/java.io.tmpdir/jetty-localhost-33481-hadoop-hdfs-3_4_1-tests_jar-_-any-3793653715201274431/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T19:24:30,421 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:33481} 2024-12-04T19:24:30,421 INFO [Time-limited test {}] server.Server(415): Started @3549ms 2024-12-04T19:24:30,423 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T19:24:31,753 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/data/data4/current/BP-1762372386-172.17.0.2-1733340268509/current, will proceed with Du for space computation calculation, 2024-12-04T19:24:31,753 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/data/data1/current/BP-1762372386-172.17.0.2-1733340268509/current, will proceed with Du for space computation calculation, 2024-12-04T19:24:31,753 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/data/data2/current/BP-1762372386-172.17.0.2-1733340268509/current, will proceed with Du for space computation calculation, 2024-12-04T19:24:31,753 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/data/data3/current/BP-1762372386-172.17.0.2-1733340268509/current, will proceed with Du for space computation calculation, 2024-12-04T19:24:31,779 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T19:24:31,779 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T19:24:31,788 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/data/data5/current/BP-1762372386-172.17.0.2-1733340268509/current, will proceed with Du for space computation calculation, 2024-12-04T19:24:31,788 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/data/data6/current/BP-1762372386-172.17.0.2-1733340268509/current, will proceed with Du for space computation calculation, 2024-12-04T19:24:31,810 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T19:24:31,826 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x609b457ddae72132 with lease ID 0x3a9dc91c2bb6678b: Processing first storage report for DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a from datanode DatanodeRegistration(127.0.0.1:41199, datanodeUuid=b04cbcf5-11a1-42aa-a434-eab201f64137, infoPort=42335, infoSecurePort=0, ipcPort=37971, storageInfo=lv=-57;cid=testClusterID;nsid=586939343;c=1733340268509) 2024-12-04T19:24:31,827 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x609b457ddae72132 with lease ID 0x3a9dc91c2bb6678b: from storage DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a node DatanodeRegistration(127.0.0.1:41199, datanodeUuid=b04cbcf5-11a1-42aa-a434-eab201f64137, infoPort=42335, infoSecurePort=0, ipcPort=37971, storageInfo=lv=-57;cid=testClusterID;nsid=586939343;c=1733340268509), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T19:24:31,827 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x800e7c6253240e54 with lease ID 0x3a9dc91c2bb6678c: Processing first storage report for DS-e5ebbdb3-f383-40bf-b110-a17e202087e2 from datanode DatanodeRegistration(127.0.0.1:44885, datanodeUuid=5808b0f9-ae19-459e-9dd8-c71922061aec, infoPort=37675, infoSecurePort=0, ipcPort=46227, storageInfo=lv=-57;cid=testClusterID;nsid=586939343;c=1733340268509) 2024-12-04T19:24:31,827 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x800e7c6253240e54 with lease ID 0x3a9dc91c2bb6678c: from storage DS-e5ebbdb3-f383-40bf-b110-a17e202087e2 node DatanodeRegistration(127.0.0.1:44885, datanodeUuid=5808b0f9-ae19-459e-9dd8-c71922061aec, infoPort=37675, infoSecurePort=0, ipcPort=46227, storageInfo=lv=-57;cid=testClusterID;nsid=586939343;c=1733340268509), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T19:24:31,827 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x96876731c315fd05 with lease ID 0x3a9dc91c2bb6678a: Processing first storage report for DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac from datanode DatanodeRegistration(127.0.0.1:38107, datanodeUuid=1e307788-1f8e-48c0-8bf9-4e0292381e9c, infoPort=33421, infoSecurePort=0, ipcPort=36727, storageInfo=lv=-57;cid=testClusterID;nsid=586939343;c=1733340268509) 2024-12-04T19:24:31,827 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x96876731c315fd05 with lease ID 0x3a9dc91c2bb6678a: from storage DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac node DatanodeRegistration(127.0.0.1:38107, datanodeUuid=1e307788-1f8e-48c0-8bf9-4e0292381e9c, infoPort=33421, infoSecurePort=0, ipcPort=36727, storageInfo=lv=-57;cid=testClusterID;nsid=586939343;c=1733340268509), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T19:24:31,828 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x609b457ddae72132 with lease ID 0x3a9dc91c2bb6678b: Processing first storage report for DS-d1486634-d492-4101-9bdf-4d1906869049 from datanode DatanodeRegistration(127.0.0.1:41199, datanodeUuid=b04cbcf5-11a1-42aa-a434-eab201f64137, infoPort=42335, infoSecurePort=0, ipcPort=37971, storageInfo=lv=-57;cid=testClusterID;nsid=586939343;c=1733340268509) 2024-12-04T19:24:31,828 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x609b457ddae72132 with lease ID 0x3a9dc91c2bb6678b: from storage DS-d1486634-d492-4101-9bdf-4d1906869049 node DatanodeRegistration(127.0.0.1:41199, datanodeUuid=b04cbcf5-11a1-42aa-a434-eab201f64137, infoPort=42335, infoSecurePort=0, ipcPort=37971, storageInfo=lv=-57;cid=testClusterID;nsid=586939343;c=1733340268509), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T19:24:31,828 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x800e7c6253240e54 with lease ID 0x3a9dc91c2bb6678c: Processing first storage report for DS-6e5af396-0a02-4a37-b083-8f527fa1982b from datanode DatanodeRegistration(127.0.0.1:44885, datanodeUuid=5808b0f9-ae19-459e-9dd8-c71922061aec, infoPort=37675, infoSecurePort=0, ipcPort=46227, storageInfo=lv=-57;cid=testClusterID;nsid=586939343;c=1733340268509) 2024-12-04T19:24:31,828 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x800e7c6253240e54 with lease ID 0x3a9dc91c2bb6678c: from storage DS-6e5af396-0a02-4a37-b083-8f527fa1982b node DatanodeRegistration(127.0.0.1:44885, datanodeUuid=5808b0f9-ae19-459e-9dd8-c71922061aec, infoPort=37675, infoSecurePort=0, ipcPort=46227, storageInfo=lv=-57;cid=testClusterID;nsid=586939343;c=1733340268509), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T19:24:31,828 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x96876731c315fd05 with lease ID 0x3a9dc91c2bb6678a: Processing first storage report for DS-76e19aec-a735-47c9-9a74-89ab0be5ab11 from datanode DatanodeRegistration(127.0.0.1:38107, datanodeUuid=1e307788-1f8e-48c0-8bf9-4e0292381e9c, infoPort=33421, infoSecurePort=0, ipcPort=36727, storageInfo=lv=-57;cid=testClusterID;nsid=586939343;c=1733340268509) 2024-12-04T19:24:31,828 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x96876731c315fd05 with lease ID 0x3a9dc91c2bb6678a: from storage DS-76e19aec-a735-47c9-9a74-89ab0be5ab11 node DatanodeRegistration(127.0.0.1:38107, datanodeUuid=1e307788-1f8e-48c0-8bf9-4e0292381e9c, infoPort=33421, infoSecurePort=0, ipcPort=36727, storageInfo=lv=-57;cid=testClusterID;nsid=586939343;c=1733340268509), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T19:24:31,847 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1 2024-12-04T19:24:31,904 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/zookeeper_0, clientPort=61732, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T19:24:31,920 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61732 2024-12-04T19:24:31,932 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:24:31,935 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:24:32,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741825_1001 (size=7) 2024-12-04T19:24:32,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741825_1001 (size=7) 2024-12-04T19:24:32,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741825_1001 (size=7) 2024-12-04T19:24:32,530 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890 with version=8 2024-12-04T19:24:32,531 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/hbase-staging 2024-12-04T19:24:32,811 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9ba2734cd39f:0 server-side Connection retries=45 2024-12-04T19:24:32,819 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T19:24:32,820 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T19:24:32,824 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T19:24:32,824 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T19:24:32,825 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T19:24:32,934 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T19:24:32,983 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-04T19:24:32,991 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-04T19:24:32,994 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T19:24:33,016 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 66249 (auto-detected) 2024-12-04T19:24:33,017 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-04T19:24:33,033 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44405 2024-12-04T19:24:33,051 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44405 connecting to ZooKeeper ensemble=127.0.0.1:61732 2024-12-04T19:24:33,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:444050x0, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T19:24:33,178 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44405-0x101a1cd90bd0000 connected 2024-12-04T19:24:33,274 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:24:33,280 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:24:33,288 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T19:24:33,292 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890, hbase.cluster.distributed=false 2024-12-04T19:24:33,312 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T19:24:33,316 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44405 2024-12-04T19:24:33,316 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44405 2024-12-04T19:24:33,320 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44405 2024-12-04T19:24:33,321 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44405 2024-12-04T19:24:33,321 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44405 2024-12-04T19:24:33,414 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9ba2734cd39f:0 server-side Connection retries=45 2024-12-04T19:24:33,416 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T19:24:33,416 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T19:24:33,416 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T19:24:33,416 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T19:24:33,416 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T19:24:33,419 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T19:24:33,421 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T19:24:33,422 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42371 2024-12-04T19:24:33,424 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42371 connecting to ZooKeeper ensemble=127.0.0.1:61732 2024-12-04T19:24:33,425 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:24:33,430 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:24:33,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:423710x0, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T19:24:33,446 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:423710x0, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T19:24:33,446 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42371-0x101a1cd90bd0001 connected 2024-12-04T19:24:33,450 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T19:24:33,458 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T19:24:33,461 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T19:24:33,466 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T19:24:33,466 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42371 2024-12-04T19:24:33,467 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42371 2024-12-04T19:24:33,467 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42371 2024-12-04T19:24:33,468 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42371 2024-12-04T19:24:33,469 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42371 2024-12-04T19:24:33,484 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9ba2734cd39f:0 server-side Connection retries=45 2024-12-04T19:24:33,484 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T19:24:33,484 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T19:24:33,484 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T19:24:33,485 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T19:24:33,485 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T19:24:33,485 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T19:24:33,485 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T19:24:33,486 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46195 2024-12-04T19:24:33,488 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46195 connecting to ZooKeeper ensemble=127.0.0.1:61732 2024-12-04T19:24:33,489 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:24:33,491 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:24:33,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:461950x0, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T19:24:33,509 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:461950x0, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T19:24:33,509 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46195-0x101a1cd90bd0002 connected 2024-12-04T19:24:33,510 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T19:24:33,511 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T19:24:33,511 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46195-0x101a1cd90bd0002, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T19:24:33,513 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46195-0x101a1cd90bd0002, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T19:24:33,517 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46195 2024-12-04T19:24:33,517 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46195 2024-12-04T19:24:33,518 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46195 2024-12-04T19:24:33,518 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46195 2024-12-04T19:24:33,519 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46195 2024-12-04T19:24:33,534 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9ba2734cd39f:0 server-side Connection retries=45 2024-12-04T19:24:33,534 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T19:24:33,534 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T19:24:33,534 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T19:24:33,534 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T19:24:33,535 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T19:24:33,535 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T19:24:33,535 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T19:24:33,536 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38513 2024-12-04T19:24:33,537 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38513 connecting to ZooKeeper ensemble=127.0.0.1:61732 2024-12-04T19:24:33,538 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:24:33,541 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:24:33,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:385130x0, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T19:24:33,555 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38513-0x101a1cd90bd0003 connected 2024-12-04T19:24:33,555 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T19:24:33,556 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T19:24:33,557 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T19:24:33,558 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T19:24:33,560 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T19:24:33,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38513 2024-12-04T19:24:33,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38513 2024-12-04T19:24:33,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38513 2024-12-04T19:24:33,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38513 2024-12-04T19:24:33,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38513 2024-12-04T19:24:33,578 DEBUG [M:0;9ba2734cd39f:44405 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9ba2734cd39f:44405 2024-12-04T19:24:33,579 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9ba2734cd39f,44405,1733340272667 2024-12-04T19:24:33,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T19:24:33,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T19:24:33,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T19:24:33,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46195-0x101a1cd90bd0002, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T19:24:33,599 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9ba2734cd39f,44405,1733340272667 2024-12-04T19:24:33,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:33,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46195-0x101a1cd90bd0002, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T19:24:33,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T19:24:33,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T19:24:33,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:33,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:33,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46195-0x101a1cd90bd0002, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:33,630 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T19:24:33,631 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9ba2734cd39f,44405,1733340272667 from backup master directory 2024-12-04T19:24:33,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46195-0x101a1cd90bd0002, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T19:24:33,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9ba2734cd39f,44405,1733340272667 2024-12-04T19:24:33,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T19:24:33,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T19:24:33,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T19:24:33,646 WARN [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T19:24:33,646 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9ba2734cd39f,44405,1733340272667 2024-12-04T19:24:33,649 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-04T19:24:33,651 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-04T19:24:33,707 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/hbase.id] with ID: ad1ba118-c3ab-45fe-b5d1-9fce4b581c96 2024-12-04T19:24:33,707 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/.tmp/hbase.id 2024-12-04T19:24:33,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741826_1002 (size=42) 2024-12-04T19:24:33,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741826_1002 (size=42) 2024-12-04T19:24:33,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741826_1002 (size=42) 2024-12-04T19:24:33,722 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/.tmp/hbase.id]:[hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/hbase.id] 2024-12-04T19:24:33,765 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T19:24:33,770 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T19:24:33,786 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-04T19:24:33,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:33,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:33,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46195-0x101a1cd90bd0002, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:33,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:33,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741827_1003 (size=196) 2024-12-04T19:24:33,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741827_1003 (size=196) 2024-12-04T19:24:33,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741827_1003 (size=196) 2024-12-04T19:24:33,830 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T19:24:33,832 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T19:24:33,837 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:33,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741828_1004 (size=1189) 2024-12-04T19:24:33,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741828_1004 (size=1189) 2024-12-04T19:24:33,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741828_1004 (size=1189) 2024-12-04T19:24:33,881 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/data/master/store 2024-12-04T19:24:33,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741829_1005 (size=34) 2024-12-04T19:24:33,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741829_1005 (size=34) 2024-12-04T19:24:33,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741829_1005 (size=34) 2024-12-04T19:24:33,905 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-04T19:24:33,910 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:33,911 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T19:24:33,911 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:24:33,912 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:24:33,914 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T19:24:33,914 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:24:33,914 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:24:33,915 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733340273911Disabling compacts and flushes for region at 1733340273911Disabling writes for close at 1733340273914 (+3 ms)Writing region close event to WAL at 1733340273914Closed at 1733340273914 2024-12-04T19:24:33,918 WARN [master/9ba2734cd39f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/data/master/store/.initializing 2024-12-04T19:24:33,918 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/WALs/9ba2734cd39f,44405,1733340272667 2024-12-04T19:24:33,926 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:24:33,941 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ba2734cd39f%2C44405%2C1733340272667, suffix=, logDir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/WALs/9ba2734cd39f,44405,1733340272667, archiveDir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/oldWALs, maxLogs=10 2024-12-04T19:24:33,976 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/WALs/9ba2734cd39f,44405,1733340272667/9ba2734cd39f%2C44405%2C1733340272667.1733340273947, exclude list is [], retry=0 2024-12-04T19:24:33,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:24:34,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:34,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:34,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:34,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-04T19:24:34,032 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-12-04T19:24:34,084 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/WALs/9ba2734cd39f,44405,1733340272667/9ba2734cd39f%2C44405%2C1733340272667.1733340273947 2024-12-04T19:24:34,085 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:33421:33421)] 2024-12-04T19:24:34,085 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:24:34,086 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:34,088 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:24:34,089 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:24:34,120 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:24:34,143 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T19:24:34,146 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:34,148 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:24:34,149 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:24:34,153 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T19:24:34,153 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:34,154 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:34,154 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:24:34,157 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T19:24:34,157 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:34,158 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:34,158 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:24:34,161 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T19:24:34,161 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:34,162 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:34,163 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:24:34,166 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:24:34,168 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:24:34,175 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:24:34,175 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:24:34,178 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T19:24:34,182 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T19:24:34,186 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:24:34,187 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62786063, jitterRate=-0.06441475450992584}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T19:24:34,192 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733340274100Initializing all the Stores at 1733340274102 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733340274102Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340274103 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340274103Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340274103Cleaning up temporary data from old regions at 1733340274175 (+72 ms)Region opened successfully at 1733340274192 (+17 ms) 2024-12-04T19:24:34,193 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T19:24:34,223 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36d666ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9ba2734cd39f/172.17.0.2:0 2024-12-04T19:24:34,249 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T19:24:34,258 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T19:24:34,258 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T19:24:34,261 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T19:24:34,262 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-04T19:24:34,266 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-04T19:24:34,266 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T19:24:34,289 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T19:24:34,296 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T19:24:34,343 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T19:24:34,347 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T19:24:34,350 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T19:24:34,361 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T19:24:34,364 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T19:24:34,367 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T19:24:34,375 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T19:24:34,376 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T19:24:34,386 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T19:24:34,406 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T19:24:34,417 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T19:24:34,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T19:24:34,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T19:24:34,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46195-0x101a1cd90bd0002, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T19:24:34,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T19:24:34,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:34,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46195-0x101a1cd90bd0002, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:34,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:34,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:34,432 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9ba2734cd39f,44405,1733340272667, sessionid=0x101a1cd90bd0000, setting cluster-up flag (Was=false) 2024-12-04T19:24:34,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:34,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46195-0x101a1cd90bd0002, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:34,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:34,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:34,502 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T19:24:34,507 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9ba2734cd39f,44405,1733340272667 2024-12-04T19:24:34,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46195-0x101a1cd90bd0002, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:34,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:34,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:34,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:34,565 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T19:24:34,570 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9ba2734cd39f,44405,1733340272667 2024-12-04T19:24:34,580 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T19:24:34,649 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T19:24:34,658 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T19:24:34,665 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T19:24:34,667 INFO [RS:2;9ba2734cd39f:38513 {}] regionserver.HRegionServer(746): ClusterId : ad1ba118-c3ab-45fe-b5d1-9fce4b581c96 2024-12-04T19:24:34,667 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.HRegionServer(746): ClusterId : ad1ba118-c3ab-45fe-b5d1-9fce4b581c96 2024-12-04T19:24:34,667 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.HRegionServer(746): ClusterId : ad1ba118-c3ab-45fe-b5d1-9fce4b581c96 2024-12-04T19:24:34,669 DEBUG [RS:2;9ba2734cd39f:38513 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T19:24:34,669 DEBUG [RS:0;9ba2734cd39f:42371 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T19:24:34,669 DEBUG [RS:1;9ba2734cd39f:46195 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T19:24:34,671 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9ba2734cd39f,44405,1733340272667 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T19:24:34,808 DEBUG [RS:0;9ba2734cd39f:42371 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T19:24:34,808 DEBUG [RS:2;9ba2734cd39f:38513 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T19:24:34,808 DEBUG [RS:1;9ba2734cd39f:46195 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T19:24:34,808 DEBUG [RS:0;9ba2734cd39f:42371 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T19:24:34,808 DEBUG [RS:2;9ba2734cd39f:38513 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T19:24:34,808 DEBUG [RS:1;9ba2734cd39f:46195 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T19:24:34,810 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9ba2734cd39f:0, corePoolSize=5, maxPoolSize=5 2024-12-04T19:24:34,810 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9ba2734cd39f:0, corePoolSize=5, maxPoolSize=5 2024-12-04T19:24:34,810 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9ba2734cd39f:0, corePoolSize=5, maxPoolSize=5 2024-12-04T19:24:34,810 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9ba2734cd39f:0, corePoolSize=5, maxPoolSize=5 2024-12-04T19:24:34,811 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9ba2734cd39f:0, corePoolSize=10, maxPoolSize=10 2024-12-04T19:24:34,811 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:34,811 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9ba2734cd39f:0, corePoolSize=2, maxPoolSize=2 2024-12-04T19:24:34,811 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:34,812 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733340304812 2024-12-04T19:24:34,814 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T19:24:34,815 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T19:24:34,817 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T19:24:34,817 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T19:24:34,818 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T19:24:34,818 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T19:24:34,819 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T19:24:34,819 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T19:24:34,819 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:34,822 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T19:24:34,823 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:34,824 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T19:24:34,823 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T19:24:34,824 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T19:24:34,826 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T19:24:34,827 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T19:24:34,828 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9ba2734cd39f:0:becomeActiveMaster-HFileCleaner.large.0-1733340274828,5,FailOnTimeoutGroup] 2024-12-04T19:24:34,829 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9ba2734cd39f:0:becomeActiveMaster-HFileCleaner.small.0-1733340274829,5,FailOnTimeoutGroup] 2024-12-04T19:24:34,829 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:34,830 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T19:24:34,831 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:34,832 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:34,836 DEBUG [RS:1;9ba2734cd39f:46195 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T19:24:34,836 DEBUG [RS:2;9ba2734cd39f:38513 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T19:24:34,836 DEBUG [RS:0;9ba2734cd39f:42371 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T19:24:34,836 DEBUG [RS:2;9ba2734cd39f:38513 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5868560d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9ba2734cd39f/172.17.0.2:0 2024-12-04T19:24:34,836 DEBUG [RS:0;9ba2734cd39f:42371 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b877ebf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9ba2734cd39f/172.17.0.2:0 2024-12-04T19:24:34,836 DEBUG [RS:1;9ba2734cd39f:46195 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7476fc6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9ba2734cd39f/172.17.0.2:0 2024-12-04T19:24:34,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741831_1007 (size=1321) 2024-12-04T19:24:34,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741831_1007 (size=1321) 2024-12-04T19:24:34,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741831_1007 (size=1321) 2024-12-04T19:24:34,842 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T19:24:34,842 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890 2024-12-04T19:24:34,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741832_1008 (size=32) 2024-12-04T19:24:34,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741832_1008 (size=32) 2024-12-04T19:24:34,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741832_1008 (size=32) 2024-12-04T19:24:34,857 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:34,860 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T19:24:34,862 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T19:24:34,862 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:34,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:24:34,863 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T19:24:34,866 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T19:24:34,866 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:34,867 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:24:34,868 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T19:24:34,871 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T19:24:34,871 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:34,872 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:24:34,872 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T19:24:34,876 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T19:24:34,876 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:34,877 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:24:34,877 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T19:24:34,879 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740 2024-12-04T19:24:34,879 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740 2024-12-04T19:24:34,880 DEBUG [RS:2;9ba2734cd39f:38513 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;9ba2734cd39f:38513 2024-12-04T19:24:34,882 DEBUG [RS:0;9ba2734cd39f:42371 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9ba2734cd39f:42371 2024-12-04T19:24:34,882 DEBUG [RS:1;9ba2734cd39f:46195 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;9ba2734cd39f:46195 2024-12-04T19:24:34,883 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T19:24:34,883 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T19:24:34,884 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T19:24:34,884 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T19:24:34,884 INFO [RS:2;9ba2734cd39f:38513 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T19:24:34,884 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T19:24:34,884 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T19:24:34,884 INFO [RS:2;9ba2734cd39f:38513 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T19:24:34,884 DEBUG [RS:1;9ba2734cd39f:46195 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T19:24:34,884 DEBUG [RS:0;9ba2734cd39f:42371 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T19:24:34,884 DEBUG [RS:2;9ba2734cd39f:38513 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T19:24:34,884 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T19:24:34,887 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.HRegionServer(2659): reportForDuty to master=9ba2734cd39f,44405,1733340272667 with port=46195, startcode=1733340273483 2024-12-04T19:24:34,887 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.HRegionServer(2659): reportForDuty to master=9ba2734cd39f,44405,1733340272667 with port=42371, startcode=1733340273381 2024-12-04T19:24:34,887 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T19:24:34,887 INFO [RS:2;9ba2734cd39f:38513 {}] regionserver.HRegionServer(2659): reportForDuty to master=9ba2734cd39f,44405,1733340272667 with port=38513, startcode=1733340273533 2024-12-04T19:24:34,892 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:24:34,893 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62017586, jitterRate=-0.07586595416069031}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T19:24:34,895 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733340274857Initializing all the Stores at 1733340274859 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733340274859Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733340274859Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340274859Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733340274860 (+1 ms)Cleaning up temporary data from old regions at 1733340274883 (+23 ms)Region opened successfully at 1733340274895 (+12 ms) 2024-12-04T19:24:34,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T19:24:34,896 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T19:24:34,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T19:24:34,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T19:24:34,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T19:24:34,897 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T19:24:34,897 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733340274896Disabling compacts and flushes for region at 1733340274896Disabling writes for close at 1733340274896Writing region close event to WAL at 1733340274897 (+1 ms)Closed at 1733340274897 2024-12-04T19:24:34,901 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T19:24:34,901 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T19:24:34,902 DEBUG [RS:1;9ba2734cd39f:46195 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T19:24:34,902 DEBUG [RS:0;9ba2734cd39f:42371 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T19:24:34,902 DEBUG [RS:2;9ba2734cd39f:38513 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T19:24:34,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T19:24:34,931 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T19:24:34,935 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T19:24:34,939 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39391, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T19:24:34,939 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39145, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T19:24:34,939 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36909, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T19:24:34,945 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44405 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9ba2734cd39f,42371,1733340273381 2024-12-04T19:24:34,947 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44405 {}] master.ServerManager(517): Registering regionserver=9ba2734cd39f,42371,1733340273381 2024-12-04T19:24:34,957 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44405 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9ba2734cd39f,38513,1733340273533 2024-12-04T19:24:34,957 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44405 {}] master.ServerManager(517): Registering regionserver=9ba2734cd39f,38513,1733340273533 2024-12-04T19:24:34,961 DEBUG [RS:0;9ba2734cd39f:42371 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890 2024-12-04T19:24:34,961 DEBUG [RS:0;9ba2734cd39f:42371 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33439 2024-12-04T19:24:34,961 DEBUG [RS:0;9ba2734cd39f:42371 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T19:24:34,962 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44405 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9ba2734cd39f,46195,1733340273483 2024-12-04T19:24:34,962 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44405 {}] master.ServerManager(517): Registering regionserver=9ba2734cd39f,46195,1733340273483 2024-12-04T19:24:34,962 DEBUG [RS:2;9ba2734cd39f:38513 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890 2024-12-04T19:24:34,962 DEBUG [RS:2;9ba2734cd39f:38513 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33439 2024-12-04T19:24:34,963 DEBUG [RS:2;9ba2734cd39f:38513 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T19:24:34,966 DEBUG [RS:1;9ba2734cd39f:46195 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890 2024-12-04T19:24:34,966 DEBUG [RS:1;9ba2734cd39f:46195 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33439 2024-12-04T19:24:34,966 DEBUG [RS:1;9ba2734cd39f:46195 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T19:24:35,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T19:24:35,017 DEBUG [RS:2;9ba2734cd39f:38513 {}] zookeeper.ZKUtil(111): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9ba2734cd39f,38513,1733340273533 2024-12-04T19:24:35,017 DEBUG [RS:0;9ba2734cd39f:42371 {}] zookeeper.ZKUtil(111): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9ba2734cd39f,42371,1733340273381 2024-12-04T19:24:35,018 WARN [RS:2;9ba2734cd39f:38513 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T19:24:35,018 WARN [RS:0;9ba2734cd39f:42371 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T19:24:35,018 INFO [RS:2;9ba2734cd39f:38513 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:35,018 INFO [RS:0;9ba2734cd39f:42371 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:35,018 DEBUG [RS:1;9ba2734cd39f:46195 {}] zookeeper.ZKUtil(111): regionserver:46195-0x101a1cd90bd0002, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9ba2734cd39f,46195,1733340273483 2024-12-04T19:24:35,018 DEBUG [RS:2;9ba2734cd39f:38513 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,38513,1733340273533 2024-12-04T19:24:35,018 WARN [RS:1;9ba2734cd39f:46195 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T19:24:35,018 DEBUG [RS:0;9ba2734cd39f:42371 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,42371,1733340273381 2024-12-04T19:24:35,019 INFO [RS:1;9ba2734cd39f:46195 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:35,019 DEBUG [RS:1;9ba2734cd39f:46195 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483 2024-12-04T19:24:35,021 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9ba2734cd39f,42371,1733340273381] 2024-12-04T19:24:35,021 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9ba2734cd39f,38513,1733340273533] 2024-12-04T19:24:35,021 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9ba2734cd39f,46195,1733340273483] 2024-12-04T19:24:35,044 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T19:24:35,044 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T19:24:35,044 INFO [RS:2;9ba2734cd39f:38513 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T19:24:35,056 INFO [RS:2;9ba2734cd39f:38513 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T19:24:35,056 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T19:24:35,056 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T19:24:35,061 INFO [RS:2;9ba2734cd39f:38513 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T19:24:35,062 INFO [RS:0;9ba2734cd39f:42371 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T19:24:35,062 INFO [RS:1;9ba2734cd39f:46195 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T19:24:35,062 INFO [RS:2;9ba2734cd39f:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,062 INFO [RS:1;9ba2734cd39f:46195 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,062 INFO [RS:0;9ba2734cd39f:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,063 INFO [RS:2;9ba2734cd39f:38513 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T19:24:35,063 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T19:24:35,063 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T19:24:35,068 INFO [RS:0;9ba2734cd39f:42371 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T19:24:35,068 INFO [RS:2;9ba2734cd39f:38513 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T19:24:35,070 INFO [RS:1;9ba2734cd39f:46195 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T19:24:35,070 INFO [RS:1;9ba2734cd39f:46195 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,070 INFO [RS:0;9ba2734cd39f:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,070 INFO [RS:2;9ba2734cd39f:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,071 DEBUG [RS:0;9ba2734cd39f:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,071 DEBUG [RS:2;9ba2734cd39f:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,071 DEBUG [RS:1;9ba2734cd39f:46195 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,071 DEBUG [RS:0;9ba2734cd39f:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,071 DEBUG [RS:2;9ba2734cd39f:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,071 DEBUG [RS:1;9ba2734cd39f:46195 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,071 DEBUG [RS:1;9ba2734cd39f:46195 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,071 DEBUG [RS:0;9ba2734cd39f:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,071 DEBUG [RS:2;9ba2734cd39f:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,071 DEBUG [RS:1;9ba2734cd39f:46195 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,071 DEBUG [RS:2;9ba2734cd39f:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,071 DEBUG [RS:0;9ba2734cd39f:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,071 DEBUG [RS:1;9ba2734cd39f:46195 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,071 DEBUG [RS:2;9ba2734cd39f:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,071 DEBUG [RS:1;9ba2734cd39f:46195 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0, corePoolSize=2, maxPoolSize=2 2024-12-04T19:24:35,071 DEBUG [RS:0;9ba2734cd39f:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,071 DEBUG [RS:1;9ba2734cd39f:46195 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,071 DEBUG [RS:2;9ba2734cd39f:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0, corePoolSize=2, maxPoolSize=2 2024-12-04T19:24:35,072 DEBUG [RS:0;9ba2734cd39f:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0, corePoolSize=2, maxPoolSize=2 2024-12-04T19:24:35,072 DEBUG [RS:1;9ba2734cd39f:46195 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,072 DEBUG [RS:2;9ba2734cd39f:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,072 DEBUG [RS:0;9ba2734cd39f:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,072 DEBUG [RS:1;9ba2734cd39f:46195 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,072 DEBUG [RS:2;9ba2734cd39f:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,072 DEBUG [RS:1;9ba2734cd39f:46195 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,072 DEBUG [RS:0;9ba2734cd39f:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,072 DEBUG [RS:1;9ba2734cd39f:46195 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,072 DEBUG [RS:2;9ba2734cd39f:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,072 DEBUG [RS:0;9ba2734cd39f:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,072 DEBUG [RS:1;9ba2734cd39f:46195 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,072 DEBUG [RS:2;9ba2734cd39f:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,072 DEBUG [RS:1;9ba2734cd39f:46195 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9ba2734cd39f:0, corePoolSize=3, maxPoolSize=3 2024-12-04T19:24:35,072 DEBUG [RS:0;9ba2734cd39f:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,072 DEBUG [RS:2;9ba2734cd39f:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,072 DEBUG [RS:1;9ba2734cd39f:46195 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0, corePoolSize=3, maxPoolSize=3 2024-12-04T19:24:35,072 DEBUG [RS:0;9ba2734cd39f:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,072 DEBUG [RS:2;9ba2734cd39f:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,072 DEBUG [RS:0;9ba2734cd39f:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9ba2734cd39f:0, corePoolSize=1, maxPoolSize=1 2024-12-04T19:24:35,073 DEBUG [RS:2;9ba2734cd39f:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9ba2734cd39f:0, corePoolSize=3, maxPoolSize=3 2024-12-04T19:24:35,073 DEBUG [RS:0;9ba2734cd39f:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9ba2734cd39f:0, corePoolSize=3, maxPoolSize=3 2024-12-04T19:24:35,073 DEBUG [RS:2;9ba2734cd39f:38513 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0, corePoolSize=3, maxPoolSize=3 2024-12-04T19:24:35,073 DEBUG [RS:0;9ba2734cd39f:42371 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9ba2734cd39f:0, corePoolSize=3, maxPoolSize=3 2024-12-04T19:24:35,075 INFO [RS:1;9ba2734cd39f:46195 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,075 INFO [RS:2;9ba2734cd39f:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,075 INFO [RS:0;9ba2734cd39f:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,075 INFO [RS:1;9ba2734cd39f:46195 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,075 INFO [RS:2;9ba2734cd39f:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,075 INFO [RS:0;9ba2734cd39f:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,075 INFO [RS:1;9ba2734cd39f:46195 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,075 INFO [RS:2;9ba2734cd39f:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,075 INFO [RS:0;9ba2734cd39f:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,075 INFO [RS:1;9ba2734cd39f:46195 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,076 INFO [RS:2;9ba2734cd39f:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,076 INFO [RS:0;9ba2734cd39f:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,076 INFO [RS:1;9ba2734cd39f:46195 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,076 INFO [RS:2;9ba2734cd39f:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,076 INFO [RS:0;9ba2734cd39f:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,076 INFO [RS:1;9ba2734cd39f:46195 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,46195,1733340273483-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T19:24:35,076 INFO [RS:2;9ba2734cd39f:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,38513,1733340273533-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T19:24:35,076 INFO [RS:0;9ba2734cd39f:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,42371,1733340273381-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T19:24:35,086 WARN [9ba2734cd39f:44405 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T19:24:35,101 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T19:24:35,101 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T19:24:35,101 INFO [RS:2;9ba2734cd39f:38513 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T19:24:35,104 INFO [RS:0;9ba2734cd39f:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,42371,1733340273381-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,104 INFO [RS:2;9ba2734cd39f:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,38513,1733340273533-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,104 INFO [RS:1;9ba2734cd39f:46195 {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,46195,1733340273483-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,104 INFO [RS:0;9ba2734cd39f:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,104 INFO [RS:1;9ba2734cd39f:46195 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,104 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.Replication(171): 9ba2734cd39f,42371,1733340273381 started 2024-12-04T19:24:35,104 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.Replication(171): 9ba2734cd39f,46195,1733340273483 started 2024-12-04T19:24:35,104 INFO [RS:2;9ba2734cd39f:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,105 INFO [RS:2;9ba2734cd39f:38513 {}] regionserver.Replication(171): 9ba2734cd39f,38513,1733340273533 started 2024-12-04T19:24:35,123 INFO [RS:0;9ba2734cd39f:42371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,124 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.HRegionServer(1482): Serving as 9ba2734cd39f,42371,1733340273381, RpcServer on 9ba2734cd39f/172.17.0.2:42371, sessionid=0x101a1cd90bd0001 2024-12-04T19:24:35,125 DEBUG [RS:0;9ba2734cd39f:42371 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T19:24:35,125 DEBUG [RS:0;9ba2734cd39f:42371 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9ba2734cd39f,42371,1733340273381 2024-12-04T19:24:35,125 DEBUG [RS:0;9ba2734cd39f:42371 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ba2734cd39f,42371,1733340273381' 2024-12-04T19:24:35,125 DEBUG [RS:0;9ba2734cd39f:42371 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T19:24:35,126 DEBUG [RS:0;9ba2734cd39f:42371 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T19:24:35,127 DEBUG [RS:0;9ba2734cd39f:42371 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T19:24:35,127 DEBUG [RS:0;9ba2734cd39f:42371 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T19:24:35,127 DEBUG [RS:0;9ba2734cd39f:42371 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9ba2734cd39f,42371,1733340273381 2024-12-04T19:24:35,128 DEBUG [RS:0;9ba2734cd39f:42371 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ba2734cd39f,42371,1733340273381' 2024-12-04T19:24:35,128 DEBUG [RS:0;9ba2734cd39f:42371 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T19:24:35,128 DEBUG [RS:0;9ba2734cd39f:42371 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T19:24:35,129 INFO [RS:1;9ba2734cd39f:46195 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,129 INFO [RS:2;9ba2734cd39f:38513 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,129 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.HRegionServer(1482): Serving as 9ba2734cd39f,46195,1733340273483, RpcServer on 9ba2734cd39f/172.17.0.2:46195, sessionid=0x101a1cd90bd0002 2024-12-04T19:24:35,129 INFO [RS:2;9ba2734cd39f:38513 {}] regionserver.HRegionServer(1482): Serving as 9ba2734cd39f,38513,1733340273533, RpcServer on 9ba2734cd39f/172.17.0.2:38513, sessionid=0x101a1cd90bd0003 2024-12-04T19:24:35,129 DEBUG [RS:0;9ba2734cd39f:42371 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T19:24:35,129 DEBUG [RS:2;9ba2734cd39f:38513 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T19:24:35,129 DEBUG [RS:1;9ba2734cd39f:46195 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T19:24:35,129 DEBUG [RS:2;9ba2734cd39f:38513 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9ba2734cd39f,38513,1733340273533 2024-12-04T19:24:35,129 DEBUG [RS:1;9ba2734cd39f:46195 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9ba2734cd39f,46195,1733340273483 2024-12-04T19:24:35,129 INFO [RS:0;9ba2734cd39f:42371 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T19:24:35,129 DEBUG [RS:2;9ba2734cd39f:38513 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ba2734cd39f,38513,1733340273533' 2024-12-04T19:24:35,129 DEBUG [RS:1;9ba2734cd39f:46195 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ba2734cd39f,46195,1733340273483' 2024-12-04T19:24:35,129 INFO [RS:0;9ba2734cd39f:42371 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T19:24:35,129 DEBUG [RS:2;9ba2734cd39f:38513 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T19:24:35,129 DEBUG [RS:1;9ba2734cd39f:46195 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T19:24:35,130 DEBUG [RS:1;9ba2734cd39f:46195 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T19:24:35,130 DEBUG [RS:2;9ba2734cd39f:38513 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T19:24:35,130 DEBUG [RS:2;9ba2734cd39f:38513 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T19:24:35,131 DEBUG [RS:2;9ba2734cd39f:38513 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T19:24:35,131 DEBUG [RS:1;9ba2734cd39f:46195 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T19:24:35,131 DEBUG [RS:2;9ba2734cd39f:38513 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9ba2734cd39f,38513,1733340273533 2024-12-04T19:24:35,131 DEBUG [RS:1;9ba2734cd39f:46195 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T19:24:35,131 DEBUG [RS:2;9ba2734cd39f:38513 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ba2734cd39f,38513,1733340273533' 2024-12-04T19:24:35,131 DEBUG [RS:1;9ba2734cd39f:46195 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9ba2734cd39f,46195,1733340273483 2024-12-04T19:24:35,131 DEBUG [RS:2;9ba2734cd39f:38513 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T19:24:35,131 DEBUG [RS:1;9ba2734cd39f:46195 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9ba2734cd39f,46195,1733340273483' 2024-12-04T19:24:35,131 DEBUG [RS:1;9ba2734cd39f:46195 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T19:24:35,131 DEBUG [RS:2;9ba2734cd39f:38513 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T19:24:35,131 DEBUG [RS:1;9ba2734cd39f:46195 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T19:24:35,132 DEBUG [RS:2;9ba2734cd39f:38513 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T19:24:35,132 INFO [RS:2;9ba2734cd39f:38513 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T19:24:35,132 DEBUG [RS:1;9ba2734cd39f:46195 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T19:24:35,132 INFO [RS:1;9ba2734cd39f:46195 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T19:24:35,132 INFO [RS:2;9ba2734cd39f:38513 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T19:24:35,132 INFO [RS:1;9ba2734cd39f:46195 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T19:24:35,235 INFO [RS:1;9ba2734cd39f:46195 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:24:35,235 INFO [RS:0;9ba2734cd39f:42371 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:24:35,235 INFO [RS:2;9ba2734cd39f:38513 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:24:35,239 INFO [RS:1;9ba2734cd39f:46195 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ba2734cd39f%2C46195%2C1733340273483, suffix=, logDir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483, archiveDir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/oldWALs, maxLogs=32 2024-12-04T19:24:35,239 INFO [RS:0;9ba2734cd39f:42371 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ba2734cd39f%2C42371%2C1733340273381, suffix=, logDir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,42371,1733340273381, archiveDir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/oldWALs, maxLogs=32 2024-12-04T19:24:35,239 INFO [RS:2;9ba2734cd39f:38513 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ba2734cd39f%2C38513%2C1733340273533, suffix=, logDir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,38513,1733340273533, archiveDir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/oldWALs, maxLogs=32 2024-12-04T19:24:35,254 DEBUG [RS:1;9ba2734cd39f:46195 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483/9ba2734cd39f%2C46195%2C1733340273483.1733340275242, exclude list is [], retry=0 2024-12-04T19:24:35,256 DEBUG [RS:2;9ba2734cd39f:38513 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,38513,1733340273533/9ba2734cd39f%2C38513%2C1733340273533.1733340275242, exclude list is [], retry=0 2024-12-04T19:24:35,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:35,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:35,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:35,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:35,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:35,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:35,261 DEBUG [RS:0;9ba2734cd39f:42371 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,42371,1733340273381/9ba2734cd39f%2C42371%2C1733340273381.1733340275242, exclude list is [], retry=0 2024-12-04T19:24:35,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:35,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:35,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:35,291 INFO [RS:1;9ba2734cd39f:46195 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483/9ba2734cd39f%2C46195%2C1733340273483.1733340275242 2024-12-04T19:24:35,292 DEBUG [RS:1;9ba2734cd39f:46195 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:37675:37675)] 2024-12-04T19:24:35,298 INFO [RS:2;9ba2734cd39f:38513 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,38513,1733340273533/9ba2734cd39f%2C38513%2C1733340273533.1733340275242 2024-12-04T19:24:35,299 DEBUG [RS:2;9ba2734cd39f:38513 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:42335:42335)] 2024-12-04T19:24:35,301 INFO [RS:0;9ba2734cd39f:42371 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,42371,1733340273381/9ba2734cd39f%2C42371%2C1733340273381.1733340275242 2024-12-04T19:24:35,302 DEBUG [RS:0;9ba2734cd39f:42371 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:37675:37675)] 2024-12-04T19:24:35,340 DEBUG [9ba2734cd39f:44405 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-04T19:24:35,353 DEBUG [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(204): Hosts are {9ba2734cd39f=0} racks are {/default-rack=0} 2024-12-04T19:24:35,359 DEBUG [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T19:24:35,359 DEBUG [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T19:24:35,359 DEBUG [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-04T19:24:35,359 DEBUG [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T19:24:35,359 DEBUG [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T19:24:35,359 DEBUG [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-04T19:24:35,359 INFO [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T19:24:35,359 INFO [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T19:24:35,359 INFO [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-04T19:24:35,359 DEBUG [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T19:24:35,365 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9ba2734cd39f,46195,1733340273483 2024-12-04T19:24:35,370 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9ba2734cd39f,46195,1733340273483, state=OPENING 2024-12-04T19:24:35,402 DEBUG [PEWorker-5 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T19:24:35,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:35,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46195-0x101a1cd90bd0002, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:35,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:35,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:24:35,415 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:24:35,415 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:24:35,415 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:24:35,415 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:24:35,417 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T19:24:35,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9ba2734cd39f,46195,1733340273483}] 2024-12-04T19:24:35,596 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T19:24:35,598 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48485, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T19:24:35,609 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T19:24:35,610 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:35,610 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-04T19:24:35,614 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ba2734cd39f%2C46195%2C1733340273483.meta, suffix=.meta, logDir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483, archiveDir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/oldWALs, maxLogs=32 2024-12-04T19:24:35,630 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483/9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta, exclude list is [], retry=0 2024-12-04T19:24:35,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:35,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:35,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:35,659 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483/9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta 2024-12-04T19:24:35,660 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:42335:42335)] 2024-12-04T19:24:35,660 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:24:35,661 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T19:24:35,663 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T19:24:35,668 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T19:24:35,673 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T19:24:35,673 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:35,674 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T19:24:35,674 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T19:24:35,677 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T19:24:35,679 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T19:24:35,679 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:35,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:24:35,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T19:24:35,681 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T19:24:35,681 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:35,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:24:35,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T19:24:35,684 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T19:24:35,684 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:35,684 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:24:35,685 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T19:24:35,686 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T19:24:35,686 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:35,687 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:24:35,687 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T19:24:35,688 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740 2024-12-04T19:24:35,691 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740 2024-12-04T19:24:35,693 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T19:24:35,693 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T19:24:35,694 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T19:24:35,697 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T19:24:35,698 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62312170, jitterRate=-0.07147631049156189}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T19:24:35,699 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T19:24:35,700 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733340275675Writing region info on filesystem at 1733340275675Initializing all the Stores at 1733340275677 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733340275677Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733340275677Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340275677Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733340275677Cleaning up temporary data from old regions at 1733340275694 (+17 ms)Running coprocessor post-open hooks at 1733340275699 (+5 ms)Region opened successfully at 1733340275700 (+1 ms) 2024-12-04T19:24:35,706 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733340275588 2024-12-04T19:24:35,716 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T19:24:35,716 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T19:24:35,718 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9ba2734cd39f,46195,1733340273483 2024-12-04T19:24:35,719 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9ba2734cd39f,46195,1733340273483, state=OPEN 2024-12-04T19:24:35,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T19:24:35,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T19:24:35,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T19:24:35,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46195-0x101a1cd90bd0002, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T19:24:35,771 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:24:35,771 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:24:35,771 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:24:35,772 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:24:35,772 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9ba2734cd39f,46195,1733340273483 2024-12-04T19:24:35,782 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T19:24:35,782 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9ba2734cd39f,46195,1733340273483 in 353 msec 2024-12-04T19:24:35,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T19:24:35,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 876 msec 2024-12-04T19:24:35,790 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T19:24:35,790 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T19:24:35,806 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:24:35,807 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1] 2024-12-04T19:24:35,829 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:24:35,831 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35681, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:24:35,853 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2430 sec 2024-12-04T19:24:35,854 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733340275854, completionTime=-1 2024-12-04T19:24:35,856 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-04T19:24:35,856 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T19:24:35,898 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-04T19:24:35,898 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733340335898 2024-12-04T19:24:35,898 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733340395898 2024-12-04T19:24:35,899 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 42 msec 2024-12-04T19:24:35,900 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-04T19:24:35,908 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,44405,1733340272667-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,909 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,44405,1733340272667-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,909 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,44405,1733340272667-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,910 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9ba2734cd39f:44405, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,911 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,912 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,918 DEBUG [master/9ba2734cd39f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T19:24:35,940 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.294sec 2024-12-04T19:24:35,942 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T19:24:35,944 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T19:24:35,945 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T19:24:35,945 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T19:24:35,946 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T19:24:35,947 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,44405,1733340272667-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T19:24:35,947 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,44405,1733340272667-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T19:24:35,952 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T19:24:35,953 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T19:24:35,953 INFO [master/9ba2734cd39f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9ba2734cd39f,44405,1733340272667-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:35,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d6e3190, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:24:35,991 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9ba2734cd39f,44405,-1 for getting cluster id 2024-12-04T19:24:35,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T19:24:36,001 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ad1ba118-c3ab-45fe-b5d1-9fce4b581c96' 2024-12-04T19:24:36,003 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T19:24:36,004 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ad1ba118-c3ab-45fe-b5d1-9fce4b581c96" 2024-12-04T19:24:36,004 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a1cc396, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:24:36,004 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9ba2734cd39f,44405,-1] 2024-12-04T19:24:36,007 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T19:24:36,008 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:24:36,009 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T19:24:36,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f808dbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T19:24:36,013 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:24:36,020 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1] 2024-12-04T19:24:36,021 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:24:36,024 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:24:36,043 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9ba2734cd39f,44405,1733340272667 2024-12-04T19:24:36,043 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:33439/hbase 2024-12-04T19:24:36,062 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=365, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=213, ProcessCount=11, AvailableMemoryMB=16004 2024-12-04T19:24:36,085 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:36,088 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:36,089 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:24:36,094 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-31176843, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/hregion-31176843, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:24:36,106 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-31176843/hregion-31176843.1733340276095, exclude list is [], retry=0 2024-12-04T19:24:36,110 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:36,110 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:36,111 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:36,138 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-31176843/hregion-31176843.1733340276095 2024-12-04T19:24:36,139 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:42335:42335)] 2024-12-04T19:24:36,139 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c0075f2ed9ce401efea6c14b35583acd, NAME => 'testReplayEditsWrittenIntoWAL,,1733340276085.c0075f2ed9ce401efea6c14b35583acd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33439/hbase 2024-12-04T19:24:36,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741838_1014 (size=64) 2024-12-04T19:24:36,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741838_1014 (size=64) 2024-12-04T19:24:36,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741838_1014 (size=64) 2024-12-04T19:24:36,154 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733340276085.c0075f2ed9ce401efea6c14b35583acd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:36,156 INFO [StoreOpener-c0075f2ed9ce401efea6c14b35583acd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c0075f2ed9ce401efea6c14b35583acd 2024-12-04T19:24:36,159 INFO [StoreOpener-c0075f2ed9ce401efea6c14b35583acd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0075f2ed9ce401efea6c14b35583acd columnFamilyName a 2024-12-04T19:24:36,159 DEBUG [StoreOpener-c0075f2ed9ce401efea6c14b35583acd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:36,160 INFO [StoreOpener-c0075f2ed9ce401efea6c14b35583acd-1 {}] regionserver.HStore(327): Store=c0075f2ed9ce401efea6c14b35583acd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:36,160 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c0075f2ed9ce401efea6c14b35583acd 2024-12-04T19:24:36,161 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd 2024-12-04T19:24:36,162 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd 2024-12-04T19:24:36,162 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c0075f2ed9ce401efea6c14b35583acd 2024-12-04T19:24:36,162 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c0075f2ed9ce401efea6c14b35583acd 2024-12-04T19:24:36,165 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c0075f2ed9ce401efea6c14b35583acd 2024-12-04T19:24:36,168 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:24:36,169 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c0075f2ed9ce401efea6c14b35583acd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66201523, jitterRate=-0.013520434498786926}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T19:24:36,170 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c0075f2ed9ce401efea6c14b35583acd: Writing region info on filesystem at 1733340276154Initializing all the Stores at 1733340276155 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340276155Cleaning up temporary data from old regions at 1733340276162 (+7 ms)Region opened successfully at 1733340276170 (+8 ms) 2024-12-04T19:24:36,170 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c0075f2ed9ce401efea6c14b35583acd, disabling compactions & flushes 2024-12-04T19:24:36,170 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733340276085.c0075f2ed9ce401efea6c14b35583acd. 2024-12-04T19:24:36,170 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733340276085.c0075f2ed9ce401efea6c14b35583acd. 2024-12-04T19:24:36,170 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733340276085.c0075f2ed9ce401efea6c14b35583acd. after waiting 0 ms 2024-12-04T19:24:36,170 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733340276085.c0075f2ed9ce401efea6c14b35583acd. 2024-12-04T19:24:36,171 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733340276085.c0075f2ed9ce401efea6c14b35583acd. 2024-12-04T19:24:36,171 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c0075f2ed9ce401efea6c14b35583acd: Waiting for close lock at 1733340276170Disabling compacts and flushes for region at 1733340276170Disabling writes for close at 1733340276170Writing region close event to WAL at 1733340276171 (+1 ms)Closed at 1733340276171 2024-12-04T19:24:36,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741837_1013 (size=175) 2024-12-04T19:24:36,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741837_1013 (size=175) 2024-12-04T19:24:36,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741837_1013 (size=175) 2024-12-04T19:24:36,181 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:24:36,181 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-31176843:(num 1733340276095) 2024-12-04T19:24:36,183 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-04T19:24:36,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741839_1015 (size=469) 2024-12-04T19:24:36,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741839_1015 (size=469) 2024-12-04T19:24:36,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741839_1015 (size=469) 2024-12-04T19:24:36,221 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-04T19:24:36,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741840_1016 (size=322) 2024-12-04T19:24:36,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741840_1016 (size=322) 2024-12-04T19:24:36,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741840_1016 (size=322) 2024-12-04T19:24:36,272 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal-1, size=469 (469bytes) 2024-12-04T19:24:36,273 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-04T19:24:36,273 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-04T19:24:36,273 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal-1 2024-12-04T19:24:36,278 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal-1 after 3ms 2024-12-04T19:24:36,306 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:36,306 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal-1 took 35ms 2024-12-04T19:24:36,311 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal-1 so closing down 2024-12-04T19:24:36,311 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:24:36,314 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-04T19:24:36,336 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/recovered.edits/0000000000000000001-wal-1.temp 2024-12-04T19:24:36,337 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:24:36,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741841_1017 (size=469) 2024-12-04T19:24:36,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741841_1017 (size=469) 2024-12-04T19:24:36,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741841_1017 (size=469) 2024-12-04T19:24:36,347 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-04T19:24:36,350 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/recovered.edits/0000000000000000002 2024-12-04T19:24:36,353 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 43 ms; skipped=0; WAL=hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-04T19:24:36,353 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal-1, journal: Splitting hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal-1, size=469 (469bytes) at 1733340276273Finishing writing output for hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal-1 so closing down at 1733340276311 (+38 ms)Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/recovered.edits/0000000000000000001-wal-1.temp at 1733340276336 (+25 ms)3 split writer threads finished at 1733340276337 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733340276347 (+10 ms)Rename recovered edits hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/recovered.edits/0000000000000000002 at 1733340276350 (+3 ms)Processed 2 edits across 1 Regions in 43 ms; skipped=0; WAL=hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733340276353 (+3 ms) 2024-12-04T19:24:36,365 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal-2, size=322 (322bytes) 2024-12-04T19:24:36,366 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal-2 2024-12-04T19:24:36,366 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal-2 after 0ms 2024-12-04T19:24:36,390 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:36,390 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal-2 took 25ms 2024-12-04T19:24:36,392 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal-2 so closing down 2024-12-04T19:24:36,392 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:24:36,395 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-04T19:24:36,418 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/recovered.edits/0000000000000000002-wal-2.temp 2024-12-04T19:24:36,418 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:24:36,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741842_1018 (size=322) 2024-12-04T19:24:36,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741842_1018 (size=322) 2024-12-04T19:24:36,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741842_1018 (size=322) 2024-12-04T19:24:36,427 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-04T19:24:36,453 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:36,456 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-12-04T19:24:36,457 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 67 ms; skipped=0; WAL=hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-04T19:24:36,458 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal-2, journal: Splitting hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal-2, size=322 (322bytes) at 1733340276366Finishing writing output for hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal-2 so closing down at 1733340276392 (+26 ms)Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/recovered.edits/0000000000000000002-wal-2.temp at 1733340276418 (+26 ms)3 split writer threads finished at 1733340276418Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733340276427 (+9 ms)Processed 1 edits across 1 Regions in 67 ms; skipped=0; WAL=hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733340276457 (+30 ms) 2024-12-04T19:24:36,458 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:24:36,460 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:24:36,474 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal.1733340276461, exclude list is [], retry=0 2024-12-04T19:24:36,478 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:36,478 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:36,478 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:36,503 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal.1733340276461 2024-12-04T19:24:36,503 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:42335:42335)] 2024-12-04T19:24:36,503 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c0075f2ed9ce401efea6c14b35583acd, NAME => 'testReplayEditsWrittenIntoWAL,,1733340276085.c0075f2ed9ce401efea6c14b35583acd.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:24:36,503 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733340276085.c0075f2ed9ce401efea6c14b35583acd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:36,503 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c0075f2ed9ce401efea6c14b35583acd 2024-12-04T19:24:36,503 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c0075f2ed9ce401efea6c14b35583acd 2024-12-04T19:24:36,506 INFO [StoreOpener-c0075f2ed9ce401efea6c14b35583acd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c0075f2ed9ce401efea6c14b35583acd 2024-12-04T19:24:36,508 INFO [StoreOpener-c0075f2ed9ce401efea6c14b35583acd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0075f2ed9ce401efea6c14b35583acd columnFamilyName a 2024-12-04T19:24:36,508 DEBUG [StoreOpener-c0075f2ed9ce401efea6c14b35583acd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:36,509 INFO [StoreOpener-c0075f2ed9ce401efea6c14b35583acd-1 {}] regionserver.HStore(327): Store=c0075f2ed9ce401efea6c14b35583acd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:36,509 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c0075f2ed9ce401efea6c14b35583acd 2024-12-04T19:24:36,511 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd 2024-12-04T19:24:36,514 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd 2024-12-04T19:24:36,515 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/recovered.edits/0000000000000000002 2024-12-04T19:24:36,540 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:36,546 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/recovered.edits/0000000000000000002 2024-12-04T19:24:36,550 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c0075f2ed9ce401efea6c14b35583acd 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-04T19:24:36,598 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/.tmp/a/a9d3b381146a4718971ab24fbfc88787 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733340276181/Put/seqid=0 2024-12-04T19:24:36,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741844_1020 (size=5170) 2024-12-04T19:24:36,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741844_1020 (size=5170) 2024-12-04T19:24:36,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741844_1020 (size=5170) 2024-12-04T19:24:36,610 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/.tmp/a/a9d3b381146a4718971ab24fbfc88787 2024-12-04T19:24:36,648 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/.tmp/a/a9d3b381146a4718971ab24fbfc88787 as hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/a/a9d3b381146a4718971ab24fbfc88787 2024-12-04T19:24:36,658 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/a/a9d3b381146a4718971ab24fbfc88787, entries=2, sequenceid=2, filesize=5.0 K 2024-12-04T19:24:36,663 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for c0075f2ed9ce401efea6c14b35583acd in 113ms, sequenceid=2, compaction requested=false; wal=null 2024-12-04T19:24:36,665 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/recovered.edits/0000000000000000002 2024-12-04T19:24:36,666 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c0075f2ed9ce401efea6c14b35583acd 2024-12-04T19:24:36,666 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c0075f2ed9ce401efea6c14b35583acd 2024-12-04T19:24:36,669 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c0075f2ed9ce401efea6c14b35583acd 2024-12-04T19:24:36,673 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/c0075f2ed9ce401efea6c14b35583acd/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-04T19:24:36,674 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c0075f2ed9ce401efea6c14b35583acd; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65875888, jitterRate=-0.018372774124145508}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T19:24:36,675 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c0075f2ed9ce401efea6c14b35583acd: Writing region info on filesystem at 1733340276504Initializing all the Stores at 1733340276505 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340276505Obtaining lock to block concurrent updates at 1733340276550 (+45 ms)Preparing flush snapshotting stores in c0075f2ed9ce401efea6c14b35583acd at 1733340276550Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733340276085.c0075f2ed9ce401efea6c14b35583acd., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733340276553 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733340276085.c0075f2ed9ce401efea6c14b35583acd. at 1733340276553Flushing c0075f2ed9ce401efea6c14b35583acd/a: creating writer at 1733340276554 (+1 ms)Flushing c0075f2ed9ce401efea6c14b35583acd/a: appending metadata at 1733340276590 (+36 ms)Flushing c0075f2ed9ce401efea6c14b35583acd/a: closing flushed file at 1733340276592 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3052174c: reopening flushed file at 1733340276646 (+54 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for c0075f2ed9ce401efea6c14b35583acd in 113ms, sequenceid=2, compaction requested=false; wal=null at 1733340276663 (+17 ms)Cleaning up temporary data from old regions at 1733340276666 (+3 ms)Region opened successfully at 1733340276675 (+9 ms) 2024-12-04T19:24:36,697 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=376 (was 365) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:45712 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:40802 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:40626 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:49524 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:49448 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1762372386-172.17.0.2-1733340268509:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1762372386-172.17.0.2-1733340268509:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:33439/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1762372386-172.17.0.2-1733340268509:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=213 (was 213), ProcessCount=11 (was 11), AvailableMemoryMB=15994 (was 16004) 2024-12-04T19:24:36,707 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=376, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=213, ProcessCount=11, AvailableMemoryMB=15993 2024-12-04T19:24:36,720 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:36,722 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:36,722 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:24:36,726 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-23077810, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/hregion-23077810, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:24:36,737 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-23077810/hregion-23077810.1733340276726, exclude list is [], retry=0 2024-12-04T19:24:36,740 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:36,740 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:36,741 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:36,765 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-23077810/hregion-23077810.1733340276726 2024-12-04T19:24:36,765 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:42335:42335)] 2024-12-04T19:24:36,765 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => beaf1231aeb7f5db5864d6003d0dce41, NAME => 'testReplayEditsWrittenIntoWAL,,1733340276720.beaf1231aeb7f5db5864d6003d0dce41.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33439/hbase 2024-12-04T19:24:36,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741846_1022 (size=64) 2024-12-04T19:24:36,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741846_1022 (size=64) 2024-12-04T19:24:36,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741846_1022 (size=64) 2024-12-04T19:24:36,778 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733340276720.beaf1231aeb7f5db5864d6003d0dce41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:36,780 INFO [StoreOpener-beaf1231aeb7f5db5864d6003d0dce41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region beaf1231aeb7f5db5864d6003d0dce41 2024-12-04T19:24:36,783 INFO [StoreOpener-beaf1231aeb7f5db5864d6003d0dce41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region beaf1231aeb7f5db5864d6003d0dce41 columnFamilyName a 2024-12-04T19:24:36,783 DEBUG [StoreOpener-beaf1231aeb7f5db5864d6003d0dce41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:36,784 INFO [StoreOpener-beaf1231aeb7f5db5864d6003d0dce41-1 {}] regionserver.HStore(327): Store=beaf1231aeb7f5db5864d6003d0dce41/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:36,784 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for beaf1231aeb7f5db5864d6003d0dce41 2024-12-04T19:24:36,785 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41 2024-12-04T19:24:36,785 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41 2024-12-04T19:24:36,786 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for beaf1231aeb7f5db5864d6003d0dce41 2024-12-04T19:24:36,786 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for beaf1231aeb7f5db5864d6003d0dce41 2024-12-04T19:24:36,788 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for beaf1231aeb7f5db5864d6003d0dce41 2024-12-04T19:24:36,791 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:24:36,792 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened beaf1231aeb7f5db5864d6003d0dce41; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73565597, jitterRate=0.09621281921863556}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T19:24:36,792 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for beaf1231aeb7f5db5864d6003d0dce41: Writing region info on filesystem at 1733340276778Initializing all the Stores at 1733340276779 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340276780 (+1 ms)Cleaning up temporary data from old regions at 1733340276786 (+6 ms)Region opened successfully at 1733340276792 (+6 ms) 2024-12-04T19:24:36,792 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing beaf1231aeb7f5db5864d6003d0dce41, disabling compactions & flushes 2024-12-04T19:24:36,792 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733340276720.beaf1231aeb7f5db5864d6003d0dce41. 2024-12-04T19:24:36,792 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733340276720.beaf1231aeb7f5db5864d6003d0dce41. 2024-12-04T19:24:36,792 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733340276720.beaf1231aeb7f5db5864d6003d0dce41. after waiting 0 ms 2024-12-04T19:24:36,792 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733340276720.beaf1231aeb7f5db5864d6003d0dce41. 2024-12-04T19:24:36,793 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733340276720.beaf1231aeb7f5db5864d6003d0dce41. 2024-12-04T19:24:36,793 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for beaf1231aeb7f5db5864d6003d0dce41: Waiting for close lock at 1733340276792Disabling compacts and flushes for region at 1733340276792Disabling writes for close at 1733340276792Writing region close event to WAL at 1733340276793 (+1 ms)Closed at 1733340276793 2024-12-04T19:24:36,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741845_1021 (size=175) 2024-12-04T19:24:36,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741845_1021 (size=175) 2024-12-04T19:24:36,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741845_1021 (size=175) 2024-12-04T19:24:36,799 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:24:36,799 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-23077810:(num 1733340276726) 2024-12-04T19:24:36,800 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-04T19:24:36,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741847_1023 (size=469) 2024-12-04T19:24:36,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741847_1023 (size=469) 2024-12-04T19:24:36,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741847_1023 (size=469) 2024-12-04T19:24:36,836 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-04T19:24:36,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741848_1024 (size=322) 2024-12-04T19:24:36,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741848_1024 (size=322) 2024-12-04T19:24:36,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741848_1024 (size=322) 2024-12-04T19:24:36,884 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal-2, size=322 (322bytes) 2024-12-04T19:24:36,884 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal-2 2024-12-04T19:24:36,885 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal-2 after 1ms 2024-12-04T19:24:36,909 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:36,909 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal-2 took 26ms 2024-12-04T19:24:36,911 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal-2 so closing down 2024-12-04T19:24:36,912 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:24:36,914 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-04T19:24:36,938 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/0000000000000000002-wal-2.temp 2024-12-04T19:24:36,938 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:24:36,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741849_1025 (size=322) 2024-12-04T19:24:36,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741849_1025 (size=322) 2024-12-04T19:24:36,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741849_1025 (size=322) 2024-12-04T19:24:36,948 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-04T19:24:36,950 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/0000000000000000002 2024-12-04T19:24:36,950 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 40 ms; skipped=0; WAL=hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-04T19:24:36,950 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal-2, journal: Splitting hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal-2, size=322 (322bytes) at 1733340276884Finishing writing output for hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal-2 so closing down at 1733340276911 (+27 ms)Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/0000000000000000002-wal-2.temp at 1733340276938 (+27 ms)3 split writer threads finished at 1733340276938Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733340276948 (+10 ms)Rename recovered edits hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/0000000000000000002 at 1733340276950 (+2 ms)Processed 1 edits across 1 Regions in 40 ms; skipped=0; WAL=hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733340276950 2024-12-04T19:24:36,965 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal-1, size=469 (469bytes) 2024-12-04T19:24:36,965 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal-1 2024-12-04T19:24:36,966 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal-1 after 1ms 2024-12-04T19:24:36,991 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:36,992 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal-1 took 27ms 2024-12-04T19:24:36,994 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal-1 so closing down 2024-12-04T19:24:36,994 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:24:36,996 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-04T19:24:37,019 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/0000000000000000001-wal-1.temp 2024-12-04T19:24:37,019 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:24:37,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741850_1026 (size=469) 2024-12-04T19:24:37,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741850_1026 (size=469) 2024-12-04T19:24:37,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741850_1026 (size=469) 2024-12-04T19:24:37,028 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-04T19:24:37,054 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:37,056 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/0000000000000000002, length=322 2024-12-04T19:24:37,058 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/0000000000000000002 2024-12-04T19:24:37,058 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 66 ms; skipped=0; WAL=hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-04T19:24:37,058 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal-1, journal: Splitting hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal-1, size=469 (469bytes) at 1733340276965Finishing writing output for hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal-1 so closing down at 1733340276994 (+29 ms)Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/0000000000000000001-wal-1.temp at 1733340277019 (+25 ms)3 split writer threads finished at 1733340277019Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733340277029 (+10 ms)Rename recovered edits hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/0000000000000000002 at 1733340277058 (+29 ms)Processed 2 edits across 1 Regions in 66 ms; skipped=0; WAL=hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733340277058 2024-12-04T19:24:37,058 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:24:37,061 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:24:37,073 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal.1733340277061, exclude list is [], retry=0 2024-12-04T19:24:37,076 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:37,076 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:37,077 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:37,102 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal.1733340277061 2024-12-04T19:24:37,102 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:37675:37675)] 2024-12-04T19:24:37,102 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => beaf1231aeb7f5db5864d6003d0dce41, NAME => 'testReplayEditsWrittenIntoWAL,,1733340276720.beaf1231aeb7f5db5864d6003d0dce41.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:24:37,102 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733340276720.beaf1231aeb7f5db5864d6003d0dce41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:37,103 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for beaf1231aeb7f5db5864d6003d0dce41 2024-12-04T19:24:37,103 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for beaf1231aeb7f5db5864d6003d0dce41 2024-12-04T19:24:37,104 INFO [StoreOpener-beaf1231aeb7f5db5864d6003d0dce41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region beaf1231aeb7f5db5864d6003d0dce41 2024-12-04T19:24:37,106 INFO [StoreOpener-beaf1231aeb7f5db5864d6003d0dce41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region beaf1231aeb7f5db5864d6003d0dce41 columnFamilyName a 2024-12-04T19:24:37,106 DEBUG [StoreOpener-beaf1231aeb7f5db5864d6003d0dce41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:37,107 INFO [StoreOpener-beaf1231aeb7f5db5864d6003d0dce41-1 {}] regionserver.HStore(327): Store=beaf1231aeb7f5db5864d6003d0dce41/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:37,107 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for beaf1231aeb7f5db5864d6003d0dce41 2024-12-04T19:24:37,108 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41 2024-12-04T19:24:37,110 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41 2024-12-04T19:24:37,111 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/0000000000000000002 2024-12-04T19:24:37,135 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:37,136 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/0000000000000000002 2024-12-04T19:24:37,137 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing beaf1231aeb7f5db5864d6003d0dce41 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-04T19:24:37,150 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/.tmp/a/38c68dc27fa748db85a0a3a72ef9d8ea is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733340276799/Put/seqid=0 2024-12-04T19:24:37,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741852_1028 (size=5170) 2024-12-04T19:24:37,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741852_1028 (size=5170) 2024-12-04T19:24:37,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741852_1028 (size=5170) 2024-12-04T19:24:37,158 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/.tmp/a/38c68dc27fa748db85a0a3a72ef9d8ea 2024-12-04T19:24:37,166 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/.tmp/a/38c68dc27fa748db85a0a3a72ef9d8ea as hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/a/38c68dc27fa748db85a0a3a72ef9d8ea 2024-12-04T19:24:37,174 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/a/38c68dc27fa748db85a0a3a72ef9d8ea, entries=2, sequenceid=2, filesize=5.0 K 2024-12-04T19:24:37,175 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for beaf1231aeb7f5db5864d6003d0dce41 in 39ms, sequenceid=2, compaction requested=false; wal=null 2024-12-04T19:24:37,176 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/0000000000000000002 2024-12-04T19:24:37,176 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for beaf1231aeb7f5db5864d6003d0dce41 2024-12-04T19:24:37,176 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for beaf1231aeb7f5db5864d6003d0dce41 2024-12-04T19:24:37,179 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for beaf1231aeb7f5db5864d6003d0dce41 2024-12-04T19:24:37,181 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/beaf1231aeb7f5db5864d6003d0dce41/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-04T19:24:37,183 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened beaf1231aeb7f5db5864d6003d0dce41; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60886099, jitterRate=-0.09272642433643341}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T19:24:37,184 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for beaf1231aeb7f5db5864d6003d0dce41: Writing region info on filesystem at 1733340277103Initializing all the Stores at 1733340277104 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340277104Obtaining lock to block concurrent updates at 1733340277137 (+33 ms)Preparing flush snapshotting stores in beaf1231aeb7f5db5864d6003d0dce41 at 1733340277137Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733340276720.beaf1231aeb7f5db5864d6003d0dce41., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733340277137Flushing stores of testReplayEditsWrittenIntoWAL,,1733340276720.beaf1231aeb7f5db5864d6003d0dce41. at 1733340277137Flushing beaf1231aeb7f5db5864d6003d0dce41/a: creating writer at 1733340277137Flushing beaf1231aeb7f5db5864d6003d0dce41/a: appending metadata at 1733340277149 (+12 ms)Flushing beaf1231aeb7f5db5864d6003d0dce41/a: closing flushed file at 1733340277149Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@249a9509: reopening flushed file at 1733340277165 (+16 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for beaf1231aeb7f5db5864d6003d0dce41 in 39ms, sequenceid=2, compaction requested=false; wal=null at 1733340277175 (+10 ms)Cleaning up temporary data from old regions at 1733340277176 (+1 ms)Region opened successfully at 1733340277183 (+7 ms) 2024-12-04T19:24:37,200 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=386 (was 376) Potentially hanging thread: PacketResponder: BP-1762372386-172.17.0.2-1733340268509:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1762372386-172.17.0.2-1733340268509:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:40844 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:49606 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1762372386-172.17.0.2-1733340268509:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:45784 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:40626 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:45530 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:49448 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=213 (was 213), ProcessCount=11 (was 11), AvailableMemoryMB=15982 (was 15993) 2024-12-04T19:24:37,210 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=386, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=213, ProcessCount=11, AvailableMemoryMB=15982 2024-12-04T19:24:37,224 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:37,226 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:37,227 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:24:37,230 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-66484545, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/hregion-66484545, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:24:37,241 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-66484545/hregion-66484545.1733340277231, exclude list is [], retry=0 2024-12-04T19:24:37,245 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:37,245 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:37,245 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:37,269 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-66484545/hregion-66484545.1733340277231 2024-12-04T19:24:37,270 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:33421:33421)] 2024-12-04T19:24:37,270 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 975d4d5da3c8da5c260aa19359299d9b, NAME => 'testReplayEditsWrittenIntoWAL,,1733340277225.975d4d5da3c8da5c260aa19359299d9b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33439/hbase 2024-12-04T19:24:37,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741854_1030 (size=64) 2024-12-04T19:24:37,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741854_1030 (size=64) 2024-12-04T19:24:37,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741854_1030 (size=64) 2024-12-04T19:24:37,284 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733340277225.975d4d5da3c8da5c260aa19359299d9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:37,286 INFO [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 975d4d5da3c8da5c260aa19359299d9b 2024-12-04T19:24:37,288 INFO [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 975d4d5da3c8da5c260aa19359299d9b columnFamilyName a 2024-12-04T19:24:37,288 DEBUG [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:37,289 INFO [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] regionserver.HStore(327): Store=975d4d5da3c8da5c260aa19359299d9b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:37,289 INFO [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 975d4d5da3c8da5c260aa19359299d9b 2024-12-04T19:24:37,290 INFO [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 975d4d5da3c8da5c260aa19359299d9b columnFamilyName b 2024-12-04T19:24:37,290 DEBUG [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:37,291 INFO [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] regionserver.HStore(327): Store=975d4d5da3c8da5c260aa19359299d9b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:37,291 INFO [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 975d4d5da3c8da5c260aa19359299d9b 2024-12-04T19:24:37,292 INFO [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 975d4d5da3c8da5c260aa19359299d9b columnFamilyName c 2024-12-04T19:24:37,292 DEBUG [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:37,293 INFO [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] regionserver.HStore(327): Store=975d4d5da3c8da5c260aa19359299d9b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:37,293 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 975d4d5da3c8da5c260aa19359299d9b 2024-12-04T19:24:37,294 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b 2024-12-04T19:24:37,294 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b 2024-12-04T19:24:37,295 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 975d4d5da3c8da5c260aa19359299d9b 2024-12-04T19:24:37,295 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 975d4d5da3c8da5c260aa19359299d9b 2024-12-04T19:24:37,296 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:24:37,297 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 975d4d5da3c8da5c260aa19359299d9b 2024-12-04T19:24:37,300 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:24:37,300 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 975d4d5da3c8da5c260aa19359299d9b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72061199, jitterRate=0.07379554212093353}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:24:37,301 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 975d4d5da3c8da5c260aa19359299d9b: Writing region info on filesystem at 1733340277284Initializing all the Stores at 1733340277285 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340277286 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340277286Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340277286Cleaning up temporary data from old regions at 1733340277295 (+9 ms)Region opened successfully at 1733340277300 (+5 ms) 2024-12-04T19:24:37,301 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 975d4d5da3c8da5c260aa19359299d9b, disabling compactions & flushes 2024-12-04T19:24:37,301 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733340277225.975d4d5da3c8da5c260aa19359299d9b. 2024-12-04T19:24:37,301 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733340277225.975d4d5da3c8da5c260aa19359299d9b. 2024-12-04T19:24:37,301 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733340277225.975d4d5da3c8da5c260aa19359299d9b. after waiting 0 ms 2024-12-04T19:24:37,301 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733340277225.975d4d5da3c8da5c260aa19359299d9b. 2024-12-04T19:24:37,301 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733340277225.975d4d5da3c8da5c260aa19359299d9b. 2024-12-04T19:24:37,301 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 975d4d5da3c8da5c260aa19359299d9b: Waiting for close lock at 1733340277301Disabling compacts and flushes for region at 1733340277301Disabling writes for close at 1733340277301Writing region close event to WAL at 1733340277301Closed at 1733340277301 2024-12-04T19:24:37,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741853_1029 (size=175) 2024-12-04T19:24:37,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741853_1029 (size=175) 2024-12-04T19:24:37,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741853_1029 (size=175) 2024-12-04T19:24:37,310 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:24:37,310 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-66484545:(num 1733340277231) 2024-12-04T19:24:37,310 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:24:37,312 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:24:37,328 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223/wal.1733340277313, exclude list is [], retry=0 2024-12-04T19:24:37,331 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:37,332 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:37,332 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:37,366 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223/wal.1733340277313 2024-12-04T19:24:37,367 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:33421:33421)] 2024-12-04T19:24:37,569 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223/wal.1733340277313, size=0 (0bytes) 2024-12-04T19:24:37,569 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223/wal.1733340277313 might be still open, length is 0 2024-12-04T19:24:37,569 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223/wal.1733340277313 2024-12-04T19:24:37,570 WARN [IPC Server handler 1 on default port 33439 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223/wal.1733340277313 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-04T19:24:37,570 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223/wal.1733340277313 after 1ms 2024-12-04T19:24:37,829 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:40872 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41199:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40872 dst: /127.0.0.1:41199 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41199 remote=/127.0.0.1:40872]. Total timeout mills is 60000, 59713 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:24:37,830 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:45816 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:38107:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45816 dst: /127.0.0.1:38107 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:24:37,830 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:49646 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:44885:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49646 dst: /127.0.0.1:44885 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:24:37,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741855_1032 (size=497775) 2024-12-04T19:24:37,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741855_1032 (size=497775) 2024-12-04T19:24:37,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741855_1032 (size=497775) 2024-12-04T19:24:41,191 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T19:24:41,235 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T19:24:41,571 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223/wal.1733340277313 after 4002ms 2024-12-04T19:24:41,602 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:41,603 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223/wal.1733340277313 took 4035ms 2024-12-04T19:24:41,608 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733340277313.temp 2024-12-04T19:24:41,642 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/recovered.edits/0000000000000000001-wal.1733340277313.temp 2024-12-04T19:24:41,689 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223/wal.1733340277313; continuing. 2024-12-04T19:24:41,689 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223/wal.1733340277313 so closing down 2024-12-04T19:24:41,689 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:24:41,689 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:24:41,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741856_1033 (size=497783) 2024-12-04T19:24:41,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741856_1033 (size=497783) 2024-12-04T19:24:41,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741856_1033 (size=497783) 2024-12-04T19:24:41,693 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/recovered.edits/0000000000000000001-wal.1733340277313.temp (wrote 3002 edits, skipped 0 edits in 37 ms) 2024-12-04T19:24:41,695 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/recovered.edits/0000000000000000001-wal.1733340277313.temp to hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/recovered.edits/0000000000000003002 2024-12-04T19:24:41,695 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 91 ms; skipped=0; WAL=hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223/wal.1733340277313, size=0, length=0, corrupted=false, cancelled=false 2024-12-04T19:24:41,695 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223/wal.1733340277313, journal: Splitting hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223/wal.1733340277313, size=0 (0bytes) at 1733340277569Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/recovered.edits/0000000000000000001-wal.1733340277313.temp at 1733340281642 (+4073 ms)Split 1024 edits, skipped 0 edits. at 1733340281648 (+6 ms)Split 2048 edits, skipped 0 edits. at 1733340281671 (+23 ms)Finishing writing output for hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223/wal.1733340277313 so closing down at 1733340281689 (+18 ms)3 split writer threads finished at 1733340281689Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/recovered.edits/0000000000000000001-wal.1733340277313.temp (wrote 3002 edits, skipped 0 edits in 37 ms) at 1733340281693 (+4 ms)Rename recovered edits hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/recovered.edits/0000000000000000001-wal.1733340277313.temp to hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/recovered.edits/0000000000000003002 at 1733340281695 (+2 ms)Processed 3002 edits across 1 Regions in 91 ms; skipped=0; WAL=hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223/wal.1733340277313, size=0, length=0, corrupted=false, cancelled=false at 1733340281695 2024-12-04T19:24:41,697 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223/wal.1733340277313 to hdfs://localhost:33439/hbase/oldWALs/wal.1733340277313 2024-12-04T19:24:41,699 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/recovered.edits/0000000000000003002 2024-12-04T19:24:41,699 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:24:41,702 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:24:41,713 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223/wal.1733340281702, exclude list is [], retry=0 2024-12-04T19:24:41,716 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:41,716 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:41,716 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:41,739 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223/wal.1733340281702 2024-12-04T19:24:41,739 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:33421:33421)] 2024-12-04T19:24:41,739 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733340277225.975d4d5da3c8da5c260aa19359299d9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:41,741 INFO [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 975d4d5da3c8da5c260aa19359299d9b 2024-12-04T19:24:41,742 INFO [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 975d4d5da3c8da5c260aa19359299d9b columnFamilyName a 2024-12-04T19:24:41,743 DEBUG [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:41,743 INFO [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] regionserver.HStore(327): Store=975d4d5da3c8da5c260aa19359299d9b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:41,743 INFO [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 975d4d5da3c8da5c260aa19359299d9b 2024-12-04T19:24:41,744 INFO [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 975d4d5da3c8da5c260aa19359299d9b columnFamilyName b 2024-12-04T19:24:41,744 DEBUG [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:41,745 INFO [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] regionserver.HStore(327): Store=975d4d5da3c8da5c260aa19359299d9b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:41,745 INFO [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 975d4d5da3c8da5c260aa19359299d9b 2024-12-04T19:24:41,746 INFO [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 975d4d5da3c8da5c260aa19359299d9b columnFamilyName c 2024-12-04T19:24:41,746 DEBUG [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:41,747 INFO [StoreOpener-975d4d5da3c8da5c260aa19359299d9b-1 {}] regionserver.HStore(327): Store=975d4d5da3c8da5c260aa19359299d9b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:41,747 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 975d4d5da3c8da5c260aa19359299d9b 2024-12-04T19:24:41,748 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b 2024-12-04T19:24:41,750 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b 2024-12-04T19:24:41,751 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/recovered.edits/0000000000000003002 2024-12-04T19:24:41,776 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:41,803 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-04T19:24:42,137 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 975d4d5da3c8da5c260aa19359299d9b 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-04T19:24:42,168 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/.tmp/a/52296625cbf648c9851f5cee364722f3 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733340277375/Put/seqid=0 2024-12-04T19:24:42,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741858_1035 (size=50463) 2024-12-04T19:24:42,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741858_1035 (size=50463) 2024-12-04T19:24:42,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741858_1035 (size=50463) 2024-12-04T19:24:42,176 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/.tmp/a/52296625cbf648c9851f5cee364722f3 2024-12-04T19:24:42,183 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/.tmp/a/52296625cbf648c9851f5cee364722f3 as hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/a/52296625cbf648c9851f5cee364722f3 2024-12-04T19:24:42,191 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/a/52296625cbf648c9851f5cee364722f3, entries=754, sequenceid=754, filesize=49.3 K 2024-12-04T19:24:42,191 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 975d4d5da3c8da5c260aa19359299d9b in 55ms, sequenceid=754, compaction requested=false; wal=null 2024-12-04T19:24:42,206 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-04T19:24:42,207 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 975d4d5da3c8da5c260aa19359299d9b 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-04T19:24:42,214 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/.tmp/a/74d8edb8f8b0444d85d5642a548d2df7 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733340277428/Put/seqid=0 2024-12-04T19:24:42,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741859_1036 (size=20072) 2024-12-04T19:24:42,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741859_1036 (size=20072) 2024-12-04T19:24:42,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741859_1036 (size=20072) 2024-12-04T19:24:42,223 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/.tmp/a/74d8edb8f8b0444d85d5642a548d2df7 2024-12-04T19:24:42,245 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/.tmp/b/5b9fc7eac5ab4f3e95e60c2bb25c4e4f is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733340277454/Put/seqid=0 2024-12-04T19:24:42,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741860_1037 (size=35835) 2024-12-04T19:24:42,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741860_1037 (size=35835) 2024-12-04T19:24:42,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741860_1037 (size=35835) 2024-12-04T19:24:42,254 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/.tmp/b/5b9fc7eac5ab4f3e95e60c2bb25c4e4f 2024-12-04T19:24:42,262 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/.tmp/a/74d8edb8f8b0444d85d5642a548d2df7 as hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/a/74d8edb8f8b0444d85d5642a548d2df7 2024-12-04T19:24:42,271 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/a/74d8edb8f8b0444d85d5642a548d2df7, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-04T19:24:42,273 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/.tmp/b/5b9fc7eac5ab4f3e95e60c2bb25c4e4f as hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/b/5b9fc7eac5ab4f3e95e60c2bb25c4e4f 2024-12-04T19:24:42,279 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/b/5b9fc7eac5ab4f3e95e60c2bb25c4e4f, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-04T19:24:42,279 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 975d4d5da3c8da5c260aa19359299d9b in 73ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-04T19:24:42,291 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-04T19:24:42,291 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 975d4d5da3c8da5c260aa19359299d9b 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-04T19:24:42,297 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/.tmp/b/00cbc17c008d41b9a69971ce2149c29d is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733340277477/Put/seqid=0 2024-12-04T19:24:42,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741861_1038 (size=35082) 2024-12-04T19:24:42,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741861_1038 (size=35082) 2024-12-04T19:24:42,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741861_1038 (size=35082) 2024-12-04T19:24:42,307 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/.tmp/b/00cbc17c008d41b9a69971ce2149c29d 2024-12-04T19:24:42,329 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/.tmp/c/cf02515292b544219b58717111862c2f is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733340277508/Put/seqid=0 2024-12-04T19:24:42,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741862_1039 (size=20825) 2024-12-04T19:24:42,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741862_1039 (size=20825) 2024-12-04T19:24:42,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741862_1039 (size=20825) 2024-12-04T19:24:42,337 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/.tmp/c/cf02515292b544219b58717111862c2f 2024-12-04T19:24:42,343 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/.tmp/b/00cbc17c008d41b9a69971ce2149c29d as hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/b/00cbc17c008d41b9a69971ce2149c29d 2024-12-04T19:24:42,351 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/b/00cbc17c008d41b9a69971ce2149c29d, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-04T19:24:42,353 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/.tmp/c/cf02515292b544219b58717111862c2f as hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/c/cf02515292b544219b58717111862c2f 2024-12-04T19:24:42,359 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/c/cf02515292b544219b58717111862c2f, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-04T19:24:42,360 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 975d4d5da3c8da5c260aa19359299d9b in 69ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-04T19:24:42,371 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733340277540/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733340277225.975d4d5da3c8da5c260aa19359299d9b. 2024-12-04T19:24:42,374 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/recovered.edits/0000000000000003002 2024-12-04T19:24:42,375 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-04T19:24:42,375 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 975d4d5da3c8da5c260aa19359299d9b 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-04T19:24:42,385 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/.tmp/c/b98ddb4da3584fb399a1653f1c74305c is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733340277514/Put/seqid=0 2024-12-04T19:24:42,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741863_1040 (size=50301) 2024-12-04T19:24:42,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741863_1040 (size=50301) 2024-12-04T19:24:42,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741863_1040 (size=50301) 2024-12-04T19:24:42,394 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/.tmp/c/b98ddb4da3584fb399a1653f1c74305c 2024-12-04T19:24:42,402 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b98ddb4da3584fb399a1653f1c74305c 2024-12-04T19:24:42,403 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/.tmp/c/b98ddb4da3584fb399a1653f1c74305c as hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/c/b98ddb4da3584fb399a1653f1c74305c 2024-12-04T19:24:42,410 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b98ddb4da3584fb399a1653f1c74305c 2024-12-04T19:24:42,410 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/c/b98ddb4da3584fb399a1653f1c74305c, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-04T19:24:42,411 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 975d4d5da3c8da5c260aa19359299d9b in 36ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-04T19:24:42,412 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/recovered.edits/0000000000000003002 2024-12-04T19:24:42,413 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 975d4d5da3c8da5c260aa19359299d9b 2024-12-04T19:24:42,413 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 975d4d5da3c8da5c260aa19359299d9b 2024-12-04T19:24:42,414 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T19:24:42,416 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 975d4d5da3c8da5c260aa19359299d9b 2024-12-04T19:24:42,419 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenIntoWAL/975d4d5da3c8da5c260aa19359299d9b/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-04T19:24:42,420 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 975d4d5da3c8da5c260aa19359299d9b; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65011114, jitterRate=-0.03125891089439392}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T19:24:42,421 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 975d4d5da3c8da5c260aa19359299d9b: Writing region info on filesystem at 1733340281740Initializing all the Stores at 1733340281741 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340281741Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340281741Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340281741Cleaning up temporary data from old regions at 1733340282413 (+672 ms)Region opened successfully at 1733340282421 (+8 ms) 2024-12-04T19:24:42,486 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 975d4d5da3c8da5c260aa19359299d9b, disabling compactions & flushes 2024-12-04T19:24:42,486 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733340277225.975d4d5da3c8da5c260aa19359299d9b. 2024-12-04T19:24:42,486 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733340277225.975d4d5da3c8da5c260aa19359299d9b. 2024-12-04T19:24:42,487 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733340277225.975d4d5da3c8da5c260aa19359299d9b. after waiting 0 ms 2024-12-04T19:24:42,487 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733340277225.975d4d5da3c8da5c260aa19359299d9b. 2024-12-04T19:24:42,488 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733340277225.975d4d5da3c8da5c260aa19359299d9b. 2024-12-04T19:24:42,489 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 975d4d5da3c8da5c260aa19359299d9b: Waiting for close lock at 1733340282486Disabling compacts and flushes for region at 1733340282486Disabling writes for close at 1733340282487 (+1 ms)Writing region close event to WAL at 1733340282488 (+1 ms)Closed at 1733340282488 2024-12-04T19:24:42,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741857_1034 (size=175) 2024-12-04T19:24:42,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741857_1034 (size=175) 2024-12-04T19:24:42,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741857_1034 (size=175) 2024-12-04T19:24:42,495 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:24:42,495 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733340281702) 2024-12-04T19:24:42,509 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=400 (was 386) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:33439 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1748288204_22 at /127.0.0.1:45828 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1748288204_22 at /127.0.0.1:49662 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:37971 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37971 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:36727 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36727 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1748288204_22 at /127.0.0.1:54132 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:33439 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1748288204_22 at /127.0.0.1:33220 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=196 (was 213), ProcessCount=11 (was 11), AvailableMemoryMB=15892 (was 15982) 2024-12-04T19:24:42,520 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=400, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=196, ProcessCount=11, AvailableMemoryMB=15890 2024-12-04T19:24:42,535 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:42,539 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:42,541 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:24:42,544 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-79019476, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/hregion-79019476, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:24:42,557 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-79019476/hregion-79019476.1733340282545, exclude list is [], retry=0 2024-12-04T19:24:42,560 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:42,560 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:42,561 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:42,586 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-79019476/hregion-79019476.1733340282545 2024-12-04T19:24:42,586 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:37675:37675)] 2024-12-04T19:24:42,586 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f1a96402ec374a06b700e32c2297b4f7, NAME => 'test2727,,1733340282537.f1a96402ec374a06b700e32c2297b4f7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33439/hbase 2024-12-04T19:24:42,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741865_1042 (size=43) 2024-12-04T19:24:42,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741865_1042 (size=43) 2024-12-04T19:24:42,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741865_1042 (size=43) 2024-12-04T19:24:42,601 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733340282537.f1a96402ec374a06b700e32c2297b4f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:42,603 INFO [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f1a96402ec374a06b700e32c2297b4f7 2024-12-04T19:24:42,605 INFO [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1a96402ec374a06b700e32c2297b4f7 columnFamilyName a 2024-12-04T19:24:42,605 DEBUG [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:42,605 INFO [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] regionserver.HStore(327): Store=f1a96402ec374a06b700e32c2297b4f7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:42,606 INFO [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f1a96402ec374a06b700e32c2297b4f7 2024-12-04T19:24:42,607 INFO [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1a96402ec374a06b700e32c2297b4f7 columnFamilyName b 2024-12-04T19:24:42,607 DEBUG [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:42,608 INFO [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] regionserver.HStore(327): Store=f1a96402ec374a06b700e32c2297b4f7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:42,608 INFO [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f1a96402ec374a06b700e32c2297b4f7 2024-12-04T19:24:42,610 INFO [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1a96402ec374a06b700e32c2297b4f7 columnFamilyName c 2024-12-04T19:24:42,610 DEBUG [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:42,611 INFO [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] regionserver.HStore(327): Store=f1a96402ec374a06b700e32c2297b4f7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:42,611 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f1a96402ec374a06b700e32c2297b4f7 2024-12-04T19:24:42,612 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7 2024-12-04T19:24:42,612 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7 2024-12-04T19:24:42,613 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f1a96402ec374a06b700e32c2297b4f7 2024-12-04T19:24:42,613 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f1a96402ec374a06b700e32c2297b4f7 2024-12-04T19:24:42,614 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:24:42,616 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f1a96402ec374a06b700e32c2297b4f7 2024-12-04T19:24:42,619 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:24:42,620 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f1a96402ec374a06b700e32c2297b4f7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59651642, jitterRate=-0.111121267080307}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:24:42,622 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f1a96402ec374a06b700e32c2297b4f7: Writing region info on filesystem at 1733340282601Initializing all the Stores at 1733340282602 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340282602Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340282603 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340282603Cleaning up temporary data from old regions at 1733340282613 (+10 ms)Region opened successfully at 1733340282622 (+9 ms) 2024-12-04T19:24:42,622 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f1a96402ec374a06b700e32c2297b4f7, disabling compactions & flushes 2024-12-04T19:24:42,622 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733340282537.f1a96402ec374a06b700e32c2297b4f7. 2024-12-04T19:24:42,622 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733340282537.f1a96402ec374a06b700e32c2297b4f7. 2024-12-04T19:24:42,622 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733340282537.f1a96402ec374a06b700e32c2297b4f7. after waiting 0 ms 2024-12-04T19:24:42,622 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733340282537.f1a96402ec374a06b700e32c2297b4f7. 2024-12-04T19:24:42,623 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733340282537.f1a96402ec374a06b700e32c2297b4f7. 2024-12-04T19:24:42,623 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f1a96402ec374a06b700e32c2297b4f7: Waiting for close lock at 1733340282622Disabling compacts and flushes for region at 1733340282622Disabling writes for close at 1733340282622Writing region close event to WAL at 1733340282623 (+1 ms)Closed at 1733340282623 2024-12-04T19:24:42,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741864_1041 (size=175) 2024-12-04T19:24:42,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741864_1041 (size=175) 2024-12-04T19:24:42,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741864_1041 (size=175) 2024-12-04T19:24:42,630 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:24:42,630 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-79019476:(num 1733340282545) 2024-12-04T19:24:42,630 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:24:42,632 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:24:42,646 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282633, exclude list is [], retry=0 2024-12-04T19:24:42,649 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:42,650 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:42,650 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:42,674 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282633 2024-12-04T19:24:42,675 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:42335:42335)] 2024-12-04T19:24:42,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741866_1043 (size=371388) 2024-12-04T19:24:42,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741866_1043 (size=371388) 2024-12-04T19:24:42,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741866_1043 (size=371388) 2024-12-04T19:24:42,820 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282633, size=362.7 K (371388bytes) 2024-12-04T19:24:42,820 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282633 2024-12-04T19:24:42,821 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282633 after 1ms 2024-12-04T19:24:42,849 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:42,850 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282633 took 30ms 2024-12-04T19:24:42,855 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733340282633.temp 2024-12-04T19:24:42,893 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282633 so closing down 2024-12-04T19:24:42,893 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:24:42,896 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000000001-wal.1733340282633.temp 2024-12-04T19:24:42,917 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:24:42,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741867_1044 (size=371388) 2024-12-04T19:24:42,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741867_1044 (size=371388) 2024-12-04T19:24:42,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741867_1044 (size=371388) 2024-12-04T19:24:42,922 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000000001-wal.1733340282633.temp (wrote 3000 edits, skipped 0 edits in 21 ms) 2024-12-04T19:24:42,923 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000000001-wal.1733340282633.temp to hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000003000 2024-12-04T19:24:42,924 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 72 ms; skipped=0; WAL=hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282633, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-12-04T19:24:42,924 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282633, journal: Splitting hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282633, size=362.7 K (371388bytes) at 1733340282820Split 1024 edits, skipped 0 edits. at 1733340282867 (+47 ms)Split 2048 edits, skipped 0 edits. at 1733340282880 (+13 ms)Finishing writing output for hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282633 so closing down at 1733340282893 (+13 ms)Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000000001-wal.1733340282633.temp at 1733340282896 (+3 ms)3 split writer threads finished at 1733340282917 (+21 ms)Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000000001-wal.1733340282633.temp (wrote 3000 edits, skipped 0 edits in 21 ms) at 1733340282922 (+5 ms)Rename recovered edits hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000000001-wal.1733340282633.temp to hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000003000 at 1733340282923 (+1 ms)Processed 3000 edits across 1 Regions in 72 ms; skipped=0; WAL=hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282633, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1733340282924 (+1 ms) 2024-12-04T19:24:42,927 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282633 to hdfs://localhost:33439/hbase/oldWALs/wal.1733340282633 2024-12-04T19:24:42,928 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000003000 2024-12-04T19:24:42,928 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:24:42,931 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:24:42,949 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282931, exclude list is [], retry=0 2024-12-04T19:24:42,952 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:42,953 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:42,953 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:42,979 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282931 2024-12-04T19:24:42,979 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:33421:33421)] 2024-12-04T19:24:42,980 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T19:24:42,981 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T19:24:42,982 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T19:24:42,982 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T19:24:42,982 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T19:24:42,983 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-04T19:24:42,983 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-04T19:24:42,983 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-04T19:24:43,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741868_1045 (size=371515) 2024-12-04T19:24:43,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741868_1045 (size=371515) 2024-12-04T19:24:43,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741868_1045 (size=371515) 2024-12-04T19:24:43,178 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282931, size=362.8 K (371515bytes) 2024-12-04T19:24:43,179 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282931 2024-12-04T19:24:43,179 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282931 after 0ms 2024-12-04T19:24:43,207 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:43,208 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282931 took 30ms 2024-12-04T19:24:43,212 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733340282931.temp 2024-12-04T19:24:43,270 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282931 so closing down 2024-12-04T19:24:43,270 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:24:43,274 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000003001-wal.1733340282931.temp 2024-12-04T19:24:43,291 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:24:43,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741869_1046 (size=371515) 2024-12-04T19:24:43,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741869_1046 (size=371515) 2024-12-04T19:24:43,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741869_1046 (size=371515) 2024-12-04T19:24:43,300 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000003001-wal.1733340282931.temp (wrote 3000 edits, skipped 0 edits in 11 ms) 2024-12-04T19:24:43,303 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000003001-wal.1733340282931.temp to hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000006000 2024-12-04T19:24:43,303 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 94 ms; skipped=0; WAL=hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282931, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-12-04T19:24:43,303 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282931, journal: Splitting hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282931, size=362.8 K (371515bytes) at 1733340283179Split 1024 edits, skipped 0 edits. at 1733340283223 (+44 ms)Split 2048 edits, skipped 0 edits. at 1733340283261 (+38 ms)Finishing writing output for hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282931 so closing down at 1733340283270 (+9 ms)Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000003001-wal.1733340282931.temp at 1733340283274 (+4 ms)3 split writer threads finished at 1733340283291 (+17 ms)Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000003001-wal.1733340282931.temp (wrote 3000 edits, skipped 0 edits in 11 ms) at 1733340283301 (+10 ms)Rename recovered edits hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000003001-wal.1733340282931.temp to hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000006000 at 1733340283303 (+2 ms)Processed 3000 edits across 1 Regions in 94 ms; skipped=0; WAL=hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282931, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1733340283303 2024-12-04T19:24:43,305 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340282931 to hdfs://localhost:33439/hbase/oldWALs/wal.1733340282931 2024-12-04T19:24:43,306 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000006000 2024-12-04T19:24:43,306 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:24:43,310 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/test2727-manual,16010,1733340282534, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:24:43,328 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340283311, exclude list is [], retry=0 2024-12-04T19:24:43,332 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:43,332 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:43,332 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:43,357 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340283311 2024-12-04T19:24:43,358 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:37675:37675)] 2024-12-04T19:24:43,358 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f1a96402ec374a06b700e32c2297b4f7, NAME => 'test2727,,1733340282537.f1a96402ec374a06b700e32c2297b4f7.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:24:43,358 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733340282537.f1a96402ec374a06b700e32c2297b4f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:43,358 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f1a96402ec374a06b700e32c2297b4f7 2024-12-04T19:24:43,358 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f1a96402ec374a06b700e32c2297b4f7 2024-12-04T19:24:43,360 INFO [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f1a96402ec374a06b700e32c2297b4f7 2024-12-04T19:24:43,361 INFO [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1a96402ec374a06b700e32c2297b4f7 columnFamilyName a 2024-12-04T19:24:43,361 DEBUG [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:43,362 INFO [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] regionserver.HStore(327): Store=f1a96402ec374a06b700e32c2297b4f7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:43,362 INFO [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f1a96402ec374a06b700e32c2297b4f7 2024-12-04T19:24:43,364 INFO [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1a96402ec374a06b700e32c2297b4f7 columnFamilyName b 2024-12-04T19:24:43,364 DEBUG [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:43,365 INFO [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] regionserver.HStore(327): Store=f1a96402ec374a06b700e32c2297b4f7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:43,365 INFO [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f1a96402ec374a06b700e32c2297b4f7 2024-12-04T19:24:43,366 INFO [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1a96402ec374a06b700e32c2297b4f7 columnFamilyName c 2024-12-04T19:24:43,366 DEBUG [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:43,367 INFO [StoreOpener-f1a96402ec374a06b700e32c2297b4f7-1 {}] regionserver.HStore(327): Store=f1a96402ec374a06b700e32c2297b4f7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:43,367 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f1a96402ec374a06b700e32c2297b4f7 2024-12-04T19:24:43,368 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7 2024-12-04T19:24:43,370 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7 2024-12-04T19:24:43,371 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000003000 2024-12-04T19:24:43,396 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:43,418 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000003000 2024-12-04T19:24:43,420 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000006000 2024-12-04T19:24:43,447 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:43,472 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000006000 2024-12-04T19:24:43,472 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f1a96402ec374a06b700e32c2297b4f7 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-04T19:24:43,497 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/.tmp/a/78e6fc8f6b414cb7b81d86327fb640d1 is 41, key is test2727/a:100/1733340282986/Put/seqid=0 2024-12-04T19:24:43,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741871_1048 (size=84227) 2024-12-04T19:24:43,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741871_1048 (size=84227) 2024-12-04T19:24:43,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741871_1048 (size=84227) 2024-12-04T19:24:43,505 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/.tmp/a/78e6fc8f6b414cb7b81d86327fb640d1 2024-12-04T19:24:43,539 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/.tmp/b/add94a5406224fcbafdb72dbea6356c9 is 41, key is test2727/b:100/1733340283054/Put/seqid=0 2024-12-04T19:24:43,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741872_1049 (size=84609) 2024-12-04T19:24:43,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741872_1049 (size=84609) 2024-12-04T19:24:43,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741872_1049 (size=84609) 2024-12-04T19:24:43,545 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/.tmp/b/add94a5406224fcbafdb72dbea6356c9 2024-12-04T19:24:43,581 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/.tmp/c/b0d13afc64854b36a39d00f4c4b1f824 is 41, key is test2727/c:100/1733340283121/Put/seqid=0 2024-12-04T19:24:43,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741873_1050 (size=84609) 2024-12-04T19:24:43,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741873_1050 (size=84609) 2024-12-04T19:24:43,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741873_1050 (size=84609) 2024-12-04T19:24:43,586 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/.tmp/c/b0d13afc64854b36a39d00f4c4b1f824 2024-12-04T19:24:43,594 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/.tmp/a/78e6fc8f6b414cb7b81d86327fb640d1 as hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/a/78e6fc8f6b414cb7b81d86327fb640d1 2024-12-04T19:24:43,601 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/a/78e6fc8f6b414cb7b81d86327fb640d1, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-04T19:24:43,602 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/.tmp/b/add94a5406224fcbafdb72dbea6356c9 as hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/b/add94a5406224fcbafdb72dbea6356c9 2024-12-04T19:24:43,609 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/b/add94a5406224fcbafdb72dbea6356c9, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-04T19:24:43,611 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/.tmp/c/b0d13afc64854b36a39d00f4c4b1f824 as hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/c/b0d13afc64854b36a39d00f4c4b1f824 2024-12-04T19:24:43,618 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/c/b0d13afc64854b36a39d00f4c4b1f824, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-04T19:24:43,619 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for f1a96402ec374a06b700e32c2297b4f7 in 147ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-04T19:24:43,620 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000003000 2024-12-04T19:24:43,621 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/0000000000000006000 2024-12-04T19:24:43,622 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f1a96402ec374a06b700e32c2297b4f7 2024-12-04T19:24:43,623 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f1a96402ec374a06b700e32c2297b4f7 2024-12-04T19:24:43,624 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:24:43,625 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f1a96402ec374a06b700e32c2297b4f7 2024-12-04T19:24:43,628 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/test2727/f1a96402ec374a06b700e32c2297b4f7/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-04T19:24:43,630 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f1a96402ec374a06b700e32c2297b4f7; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72707729, jitterRate=0.08342958986759186}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:24:43,631 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f1a96402ec374a06b700e32c2297b4f7: Writing region info on filesystem at 1733340283358Initializing all the Stores at 1733340283360 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340283360Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340283360Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340283360Obtaining lock to block concurrent updates at 1733340283472 (+112 ms)Preparing flush snapshotting stores in f1a96402ec374a06b700e32c2297b4f7 at 1733340283472Finished memstore snapshotting test2727,,1733340282537.f1a96402ec374a06b700e32c2297b4f7., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733340283473 (+1 ms)Flushing stores of test2727,,1733340282537.f1a96402ec374a06b700e32c2297b4f7. at 1733340283473Flushing f1a96402ec374a06b700e32c2297b4f7/a: creating writer at 1733340283473Flushing f1a96402ec374a06b700e32c2297b4f7/a: appending metadata at 1733340283496 (+23 ms)Flushing f1a96402ec374a06b700e32c2297b4f7/a: closing flushed file at 1733340283496Flushing f1a96402ec374a06b700e32c2297b4f7/b: creating writer at 1733340283513 (+17 ms)Flushing f1a96402ec374a06b700e32c2297b4f7/b: appending metadata at 1733340283538 (+25 ms)Flushing f1a96402ec374a06b700e32c2297b4f7/b: closing flushed file at 1733340283538Flushing f1a96402ec374a06b700e32c2297b4f7/c: creating writer at 1733340283552 (+14 ms)Flushing f1a96402ec374a06b700e32c2297b4f7/c: appending metadata at 1733340283580 (+28 ms)Flushing f1a96402ec374a06b700e32c2297b4f7/c: closing flushed file at 1733340283580Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@358ea20: reopening flushed file at 1733340283592 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fb1e34d: reopening flushed file at 1733340283601 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22e56f21: reopening flushed file at 1733340283609 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for f1a96402ec374a06b700e32c2297b4f7 in 147ms, sequenceid=6000, compaction requested=false; wal=null at 1733340283619 (+10 ms)Cleaning up temporary data from old regions at 1733340283623 (+4 ms)Region opened successfully at 1733340283631 (+8 ms) 2024-12-04T19:24:43,632 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-04T19:24:43,632 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f1a96402ec374a06b700e32c2297b4f7, disabling compactions & flushes 2024-12-04T19:24:43,632 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733340282537.f1a96402ec374a06b700e32c2297b4f7. 2024-12-04T19:24:43,632 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733340282537.f1a96402ec374a06b700e32c2297b4f7. 2024-12-04T19:24:43,633 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733340282537.f1a96402ec374a06b700e32c2297b4f7. after waiting 0 ms 2024-12-04T19:24:43,633 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733340282537.f1a96402ec374a06b700e32c2297b4f7. 2024-12-04T19:24:43,634 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733340282537.f1a96402ec374a06b700e32c2297b4f7. 2024-12-04T19:24:43,634 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f1a96402ec374a06b700e32c2297b4f7: Waiting for close lock at 1733340283632Disabling compacts and flushes for region at 1733340283632Disabling writes for close at 1733340283633 (+1 ms)Writing region close event to WAL at 1733340283634 (+1 ms)Closed at 1733340283634 2024-12-04T19:24:43,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741870_1047 (size=175) 2024-12-04T19:24:43,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741870_1047 (size=175) 2024-12-04T19:24:43,638 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1733340282534/wal.1733340283311 not finished, retry = 0 2024-12-04T19:24:43,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741870_1047 (size=175) 2024-12-04T19:24:43,745 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:24:43,745 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733340283311) 2024-12-04T19:24:43,762 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=402 (was 400) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:45828 [Waiting for operation #32] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:33344 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:54132 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=196 (was 196), ProcessCount=11 (was 11), AvailableMemoryMB=15660 (was 15890) 2024-12-04T19:24:43,772 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=402, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=196, ProcessCount=11, AvailableMemoryMB=15659 2024-12-04T19:24:43,792 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:43,800 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340283791, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:24:43,801 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733340283800 2024-12-04T19:24:43,831 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340283791/wal.1733340283800 2024-12-04T19:24:43,833 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:37675:37675)] 2024-12-04T19:24:43,834 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a2e44dfffc53914e412b23d793c9263c, NAME => 'testSequentialEditLogSeqNum,,1733340283793.a2e44dfffc53914e412b23d793c9263c.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:24:43,834 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733340283793.a2e44dfffc53914e412b23d793c9263c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:43,834 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a2e44dfffc53914e412b23d793c9263c 2024-12-04T19:24:43,834 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a2e44dfffc53914e412b23d793c9263c 2024-12-04T19:24:43,835 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:33439/hbase/data/default/testSequentialEditLogSeqNum/a2e44dfffc53914e412b23d793c9263c doesn't exist for region: a2e44dfffc53914e412b23d793c9263c on table testSequentialEditLogSeqNum 2024-12-04T19:24:43,835 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: a2e44dfffc53914e412b23d793c9263c on table testSequentialEditLogSeqNum 2024-12-04T19:24:43,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741875_1052 (size=62) 2024-12-04T19:24:43,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741875_1052 (size=62) 2024-12-04T19:24:43,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741875_1052 (size=62) 2024-12-04T19:24:44,252 INFO [StoreOpener-a2e44dfffc53914e412b23d793c9263c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a2e44dfffc53914e412b23d793c9263c 2024-12-04T19:24:44,254 INFO [StoreOpener-a2e44dfffc53914e412b23d793c9263c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2e44dfffc53914e412b23d793c9263c columnFamilyName a 2024-12-04T19:24:44,254 DEBUG [StoreOpener-a2e44dfffc53914e412b23d793c9263c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:44,255 INFO [StoreOpener-a2e44dfffc53914e412b23d793c9263c-1 {}] regionserver.HStore(327): Store=a2e44dfffc53914e412b23d793c9263c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:44,255 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a2e44dfffc53914e412b23d793c9263c 2024-12-04T19:24:44,256 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testSequentialEditLogSeqNum/a2e44dfffc53914e412b23d793c9263c 2024-12-04T19:24:44,257 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testSequentialEditLogSeqNum/a2e44dfffc53914e412b23d793c9263c 2024-12-04T19:24:44,258 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a2e44dfffc53914e412b23d793c9263c 2024-12-04T19:24:44,258 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a2e44dfffc53914e412b23d793c9263c 2024-12-04T19:24:44,261 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a2e44dfffc53914e412b23d793c9263c 2024-12-04T19:24:44,266 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testSequentialEditLogSeqNum/a2e44dfffc53914e412b23d793c9263c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:24:44,267 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a2e44dfffc53914e412b23d793c9263c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71564724, jitterRate=0.06639748811721802}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T19:24:44,268 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a2e44dfffc53914e412b23d793c9263c: Writing region info on filesystem at 1733340283834Initializing all the Stores at 1733340284251 (+417 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340284251Cleaning up temporary data from old regions at 1733340284258 (+7 ms)Region opened successfully at 1733340284268 (+10 ms) 2024-12-04T19:24:44,282 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a2e44dfffc53914e412b23d793c9263c 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-04T19:24:44,300 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testSequentialEditLogSeqNum/a2e44dfffc53914e412b23d793c9263c/.tmp/a/f69be1c9819b4a2d9668cf386ff010db is 81, key is testSequentialEditLogSeqNum/a:x0/1733340284269/Put/seqid=0 2024-12-04T19:24:44,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741876_1053 (size=5833) 2024-12-04T19:24:44,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741876_1053 (size=5833) 2024-12-04T19:24:44,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741876_1053 (size=5833) 2024-12-04T19:24:44,307 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testSequentialEditLogSeqNum/a2e44dfffc53914e412b23d793c9263c/.tmp/a/f69be1c9819b4a2d9668cf386ff010db 2024-12-04T19:24:44,313 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testSequentialEditLogSeqNum/a2e44dfffc53914e412b23d793c9263c/.tmp/a/f69be1c9819b4a2d9668cf386ff010db as hdfs://localhost:33439/hbase/data/default/testSequentialEditLogSeqNum/a2e44dfffc53914e412b23d793c9263c/a/f69be1c9819b4a2d9668cf386ff010db 2024-12-04T19:24:44,319 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testSequentialEditLogSeqNum/a2e44dfffc53914e412b23d793c9263c/a/f69be1c9819b4a2d9668cf386ff010db, entries=10, sequenceid=13, filesize=5.7 K 2024-12-04T19:24:44,321 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for a2e44dfffc53914e412b23d793c9263c in 40ms, sequenceid=13, compaction requested=false 2024-12-04T19:24:44,321 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a2e44dfffc53914e412b23d793c9263c: 2024-12-04T19:24:44,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T19:24:44,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T19:24:44,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T19:24:44,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T19:24:44,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T19:24:44,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741874_1051 (size=3256) 2024-12-04T19:24:44,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741874_1051 (size=3256) 2024-12-04T19:24:44,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741874_1051 (size=3256) 2024-12-04T19:24:44,344 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33439/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340283791/wal.1733340283800, size=3.2 K (3256bytes) 2024-12-04T19:24:44,344 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33439/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340283791/wal.1733340283800 2024-12-04T19:24:44,344 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33439/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340283791/wal.1733340283800 after 0ms 2024-12-04T19:24:44,368 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:44,368 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33439/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340283791/wal.1733340283800 took 25ms 2024-12-04T19:24:44,372 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33439/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340283791/wal.1733340283800 so closing down 2024-12-04T19:24:44,372 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:24:44,373 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733340283800.temp 2024-12-04T19:24:44,396 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testSequentialEditLogSeqNum/a2e44dfffc53914e412b23d793c9263c/recovered.edits/0000000000000000003-wal.1733340283800.temp 2024-12-04T19:24:44,397 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:24:44,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741877_1054 (size=2650) 2024-12-04T19:24:44,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741877_1054 (size=2650) 2024-12-04T19:24:44,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741877_1054 (size=2650) 2024-12-04T19:24:44,404 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testSequentialEditLogSeqNum/a2e44dfffc53914e412b23d793c9263c/recovered.edits/0000000000000000003-wal.1733340283800.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-04T19:24:44,405 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33439/hbase/data/default/testSequentialEditLogSeqNum/a2e44dfffc53914e412b23d793c9263c/recovered.edits/0000000000000000003-wal.1733340283800.temp to hdfs://localhost:33439/hbase/data/default/testSequentialEditLogSeqNum/a2e44dfffc53914e412b23d793c9263c/recovered.edits/0000000000000000020 2024-12-04T19:24:44,405 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 36 ms; skipped=2; WAL=hdfs://localhost:33439/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340283791/wal.1733340283800, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-12-04T19:24:44,405 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33439/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340283791/wal.1733340283800, journal: Splitting hdfs://localhost:33439/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340283791/wal.1733340283800, size=3.2 K (3256bytes) at 1733340284344Finishing writing output for hdfs://localhost:33439/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340283791/wal.1733340283800 so closing down at 1733340284372 (+28 ms)Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testSequentialEditLogSeqNum/a2e44dfffc53914e412b23d793c9263c/recovered.edits/0000000000000000003-wal.1733340283800.temp at 1733340284396 (+24 ms)3 split writer threads finished at 1733340284397 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testSequentialEditLogSeqNum/a2e44dfffc53914e412b23d793c9263c/recovered.edits/0000000000000000003-wal.1733340283800.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733340284404 (+7 ms)Rename recovered edits hdfs://localhost:33439/hbase/data/default/testSequentialEditLogSeqNum/a2e44dfffc53914e412b23d793c9263c/recovered.edits/0000000000000000003-wal.1733340283800.temp to hdfs://localhost:33439/hbase/data/default/testSequentialEditLogSeqNum/a2e44dfffc53914e412b23d793c9263c/recovered.edits/0000000000000000020 at 1733340284405 (+1 ms)Processed 17 edits across 1 Regions in 36 ms; skipped=2; WAL=hdfs://localhost:33439/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733340283791/wal.1733340283800, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1733340284405 2024-12-04T19:24:44,418 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=409 (was 402) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:45828 [Waiting for operation #34] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:33344 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:54132 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=180 (was 196), ProcessCount=11 (was 11), AvailableMemoryMB=15655 (was 15659) 2024-12-04T19:24:44,430 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=409, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=180, ProcessCount=11, AvailableMemoryMB=15655 2024-12-04T19:24:44,443 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:44,445 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:44,477 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:24:44,480 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-57223410, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/hregion-57223410, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:24:44,492 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-57223410/hregion-57223410.1733340284480, exclude list is [], retry=0 2024-12-04T19:24:44,495 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:44,495 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:44,495 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:44,520 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-57223410/hregion-57223410.1733340284480 2024-12-04T19:24:44,520 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:37675:37675)] 2024-12-04T19:24:44,521 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 377e6c9373fb6d955be2a9a6c7a74e08, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733340284444.377e6c9373fb6d955be2a9a6c7a74e08.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33439/hbase 2024-12-04T19:24:44,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741879_1056 (size=70) 2024-12-04T19:24:44,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741879_1056 (size=70) 2024-12-04T19:24:44,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741879_1056 (size=70) 2024-12-04T19:24:44,530 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733340284444.377e6c9373fb6d955be2a9a6c7a74e08.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:44,531 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:44,533 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 377e6c9373fb6d955be2a9a6c7a74e08 columnFamilyName a 2024-12-04T19:24:44,533 DEBUG [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:44,533 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] regionserver.HStore(327): Store=377e6c9373fb6d955be2a9a6c7a74e08/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:44,534 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:44,535 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 377e6c9373fb6d955be2a9a6c7a74e08 columnFamilyName b 2024-12-04T19:24:44,535 DEBUG [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:44,535 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] regionserver.HStore(327): Store=377e6c9373fb6d955be2a9a6c7a74e08/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:44,536 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:44,537 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 377e6c9373fb6d955be2a9a6c7a74e08 columnFamilyName c 2024-12-04T19:24:44,537 DEBUG [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:44,538 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] regionserver.HStore(327): Store=377e6c9373fb6d955be2a9a6c7a74e08/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:44,538 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:44,539 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:44,539 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:44,540 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:44,540 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:44,541 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:24:44,542 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:44,545 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:24:44,546 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 377e6c9373fb6d955be2a9a6c7a74e08; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72921858, jitterRate=0.08662036061286926}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:24:44,547 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 377e6c9373fb6d955be2a9a6c7a74e08: Writing region info on filesystem at 1733340284530Initializing all the Stores at 1733340284531 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340284531Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340284531Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340284531Cleaning up temporary data from old regions at 1733340284540 (+9 ms)Region opened successfully at 1733340284547 (+7 ms) 2024-12-04T19:24:44,547 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 377e6c9373fb6d955be2a9a6c7a74e08, disabling compactions & flushes 2024-12-04T19:24:44,547 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733340284444.377e6c9373fb6d955be2a9a6c7a74e08. 2024-12-04T19:24:44,547 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733340284444.377e6c9373fb6d955be2a9a6c7a74e08. 2024-12-04T19:24:44,547 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733340284444.377e6c9373fb6d955be2a9a6c7a74e08. after waiting 0 ms 2024-12-04T19:24:44,547 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733340284444.377e6c9373fb6d955be2a9a6c7a74e08. 2024-12-04T19:24:44,547 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733340284444.377e6c9373fb6d955be2a9a6c7a74e08. 2024-12-04T19:24:44,548 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 377e6c9373fb6d955be2a9a6c7a74e08: Waiting for close lock at 1733340284547Disabling compacts and flushes for region at 1733340284547Disabling writes for close at 1733340284547Writing region close event to WAL at 1733340284547Closed at 1733340284547 2024-12-04T19:24:44,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741878_1055 (size=175) 2024-12-04T19:24:44,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741878_1055 (size=175) 2024-12-04T19:24:44,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741878_1055 (size=175) 2024-12-04T19:24:44,553 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:24:44,553 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-57223410:(num 1733340284480) 2024-12-04T19:24:44,553 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:24:44,555 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:24:44,566 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442/wal.1733340284555, exclude list is [], retry=0 2024-12-04T19:24:44,569 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:44,570 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:44,570 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:44,595 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442/wal.1733340284555 2024-12-04T19:24:44,595 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:33421:33421)] 2024-12-04T19:24:44,596 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 377e6c9373fb6d955be2a9a6c7a74e08, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733340284444.377e6c9373fb6d955be2a9a6c7a74e08.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:24:44,596 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733340284444.377e6c9373fb6d955be2a9a6c7a74e08.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:44,596 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:44,596 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:44,598 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:44,599 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 377e6c9373fb6d955be2a9a6c7a74e08 columnFamilyName a 2024-12-04T19:24:44,599 DEBUG [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:44,599 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] regionserver.HStore(327): Store=377e6c9373fb6d955be2a9a6c7a74e08/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:44,599 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:44,600 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 377e6c9373fb6d955be2a9a6c7a74e08 columnFamilyName b 2024-12-04T19:24:44,600 DEBUG [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:44,601 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] regionserver.HStore(327): Store=377e6c9373fb6d955be2a9a6c7a74e08/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:44,601 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:44,602 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 377e6c9373fb6d955be2a9a6c7a74e08 columnFamilyName c 2024-12-04T19:24:44,602 DEBUG [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:44,602 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] regionserver.HStore(327): Store=377e6c9373fb6d955be2a9a6c7a74e08/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:44,602 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:44,603 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:44,604 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:44,605 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:44,605 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:44,606 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:24:44,607 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:44,608 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 377e6c9373fb6d955be2a9a6c7a74e08; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60372715, jitterRate=-0.1003764420747757}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:24:44,609 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 377e6c9373fb6d955be2a9a6c7a74e08: Writing region info on filesystem at 1733340284596Initializing all the Stores at 1733340284597 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340284597Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340284597Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340284597Cleaning up temporary data from old regions at 1733340284605 (+8 ms)Region opened successfully at 1733340284608 (+3 ms) 2024-12-04T19:24:44,613 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733340284612/Put/seqid=0 2024-12-04T19:24:44,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741881_1058 (size=4826) 2024-12-04T19:24:44,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741881_1058 (size=4826) 2024-12-04T19:24:44,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741881_1058 (size=4826) 2024-12-04T19:24:44,622 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33439/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 377e6c9373fb6d955be2a9a6c7a74e08/a 2024-12-04T19:24:44,629 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-04T19:24:44,630 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T19:24:44,630 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 377e6c9373fb6d955be2a9a6c7a74e08: 2024-12-04T19:24:44,631 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/a/1e80590b8a7c4c51a19e95db8bca94bd_SeqId_3_ 2024-12-04T19:24:44,632 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33439/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 377e6c9373fb6d955be2a9a6c7a74e08/a as hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/a/1e80590b8a7c4c51a19e95db8bca94bd_SeqId_3_ - updating store file list. 2024-12-04T19:24:44,637 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1e80590b8a7c4c51a19e95db8bca94bd_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-04T19:24:44,638 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/a/1e80590b8a7c4c51a19e95db8bca94bd_SeqId_3_ into 377e6c9373fb6d955be2a9a6c7a74e08/a 2024-12-04T19:24:44,638 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33439/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 377e6c9373fb6d955be2a9a6c7a74e08/a (new location: hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/a/1e80590b8a7c4c51a19e95db8bca94bd_SeqId_3_) 2024-12-04T19:24:44,673 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33439/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442/wal.1733340284555, size=0 (0bytes) 2024-12-04T19:24:44,673 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33439/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442/wal.1733340284555 might be still open, length is 0 2024-12-04T19:24:44,673 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33439/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442/wal.1733340284555 2024-12-04T19:24:44,674 WARN [IPC Server handler 1 on default port 33439 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442/wal.1733340284555 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-04T19:24:44,674 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33439/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442/wal.1733340284555 after 1ms 2024-12-04T19:24:46,816 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:33436 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41199:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33436 dst: /127.0.0.1:41199 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41199 remote=/127.0.0.1:33436]. Total timeout mills is 60000, 57827 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:24:46,817 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:51280 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:38107:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51280 dst: /127.0.0.1:38107 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:24:46,817 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:54300 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:44885:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54300 dst: /127.0.0.1:44885 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:24:46,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741880_1059 (size=606) 2024-12-04T19:24:46,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741880_1059 (size=606) 2024-12-04T19:24:46,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741880_1059 (size=606) 2024-12-04T19:24:48,676 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33439/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442/wal.1733340284555 after 4002ms 2024-12-04T19:24:48,710 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:48,710 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33439/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442/wal.1733340284555 took 4037ms 2024-12-04T19:24:48,712 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33439/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442/wal.1733340284555; continuing. 2024-12-04T19:24:48,712 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33439/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442/wal.1733340284555 so closing down 2024-12-04T19:24:48,712 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:24:48,714 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733340284555.temp 2024-12-04T19:24:48,737 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/recovered.edits/0000000000000000005-wal.1733340284555.temp 2024-12-04T19:24:48,737 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:24:48,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741882_1060 (size=329) 2024-12-04T19:24:48,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741882_1060 (size=329) 2024-12-04T19:24:48,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741882_1060 (size=329) 2024-12-04T19:24:48,744 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/recovered.edits/0000000000000000005-wal.1733340284555.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-04T19:24:48,745 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/recovered.edits/0000000000000000005-wal.1733340284555.temp to hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/recovered.edits/0000000000000000005 2024-12-04T19:24:48,745 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 35 ms; skipped=1; WAL=hdfs://localhost:33439/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442/wal.1733340284555, size=0, length=0, corrupted=false, cancelled=false 2024-12-04T19:24:48,746 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33439/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442/wal.1733340284555, journal: Splitting hdfs://localhost:33439/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442/wal.1733340284555, size=0 (0bytes) at 1733340284673Finishing writing output for hdfs://localhost:33439/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442/wal.1733340284555 so closing down at 1733340288712 (+4039 ms)Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/recovered.edits/0000000000000000005-wal.1733340284555.temp at 1733340288737 (+25 ms)3 split writer threads finished at 1733340288737Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/recovered.edits/0000000000000000005-wal.1733340284555.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733340288744 (+7 ms)Rename recovered edits hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/recovered.edits/0000000000000000005-wal.1733340284555.temp to hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/recovered.edits/0000000000000000005 at 1733340288745 (+1 ms)Processed 2 edits across 1 Regions in 35 ms; skipped=1; WAL=hdfs://localhost:33439/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442/wal.1733340284555, size=0, length=0, corrupted=false, cancelled=false at 1733340288745 2024-12-04T19:24:48,747 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33439/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442/wal.1733340284555 to hdfs://localhost:33439/hbase/oldWALs/wal.1733340284555 2024-12-04T19:24:48,748 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/recovered.edits/0000000000000000005 2024-12-04T19:24:48,748 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:24:48,750 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:24:48,763 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442/wal.1733340288750, exclude list is [], retry=0 2024-12-04T19:24:48,766 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:48,766 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:48,766 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:48,788 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442/wal.1733340288750 2024-12-04T19:24:48,789 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:33421:33421)] 2024-12-04T19:24:48,789 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 377e6c9373fb6d955be2a9a6c7a74e08, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733340284444.377e6c9373fb6d955be2a9a6c7a74e08.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:24:48,789 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733340284444.377e6c9373fb6d955be2a9a6c7a74e08.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:48,789 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:48,789 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:48,791 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:48,791 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 377e6c9373fb6d955be2a9a6c7a74e08 columnFamilyName a 2024-12-04T19:24:48,792 DEBUG [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:48,798 DEBUG [StoreFileOpener-377e6c9373fb6d955be2a9a6c7a74e08-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1e80590b8a7c4c51a19e95db8bca94bd_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-04T19:24:48,798 DEBUG [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/a/1e80590b8a7c4c51a19e95db8bca94bd_SeqId_3_ 2024-12-04T19:24:48,798 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] regionserver.HStore(327): Store=377e6c9373fb6d955be2a9a6c7a74e08/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:48,799 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:48,800 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 377e6c9373fb6d955be2a9a6c7a74e08 columnFamilyName b 2024-12-04T19:24:48,800 DEBUG [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:48,800 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] regionserver.HStore(327): Store=377e6c9373fb6d955be2a9a6c7a74e08/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:48,800 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:48,801 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 377e6c9373fb6d955be2a9a6c7a74e08 columnFamilyName c 2024-12-04T19:24:48,801 DEBUG [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:48,802 INFO [StoreOpener-377e6c9373fb6d955be2a9a6c7a74e08-1 {}] regionserver.HStore(327): Store=377e6c9373fb6d955be2a9a6c7a74e08/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:48,802 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:48,803 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:48,804 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:48,805 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/recovered.edits/0000000000000000005 2024-12-04T19:24:48,829 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:48,830 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/recovered.edits/0000000000000000005 2024-12-04T19:24:48,830 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 377e6c9373fb6d955be2a9a6c7a74e08 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-04T19:24:48,843 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/.tmp/a/558836e82a504b11ad38292039530703 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733340284642/Put/seqid=0 2024-12-04T19:24:48,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741884_1062 (size=5149) 2024-12-04T19:24:48,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741884_1062 (size=5149) 2024-12-04T19:24:48,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741884_1062 (size=5149) 2024-12-04T19:24:48,849 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/.tmp/a/558836e82a504b11ad38292039530703 2024-12-04T19:24:48,855 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/.tmp/a/558836e82a504b11ad38292039530703 as hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/a/558836e82a504b11ad38292039530703 2024-12-04T19:24:48,859 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/a/558836e82a504b11ad38292039530703, entries=1, sequenceid=5, filesize=5.0 K 2024-12-04T19:24:48,860 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 377e6c9373fb6d955be2a9a6c7a74e08 in 30ms, sequenceid=5, compaction requested=false; wal=null 2024-12-04T19:24:48,860 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/recovered.edits/0000000000000000005 2024-12-04T19:24:48,862 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:48,862 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:48,862 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:24:48,864 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 377e6c9373fb6d955be2a9a6c7a74e08 2024-12-04T19:24:48,866 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/377e6c9373fb6d955be2a9a6c7a74e08/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-04T19:24:48,867 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 377e6c9373fb6d955be2a9a6c7a74e08; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65496535, jitterRate=-0.024025574326515198}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:24:48,867 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 377e6c9373fb6d955be2a9a6c7a74e08: Writing region info on filesystem at 1733340288789Initializing all the Stores at 1733340288790 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340288790Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340288790Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340288790Obtaining lock to block concurrent updates at 1733340288830 (+40 ms)Preparing flush snapshotting stores in 377e6c9373fb6d955be2a9a6c7a74e08 at 1733340288830Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733340284444.377e6c9373fb6d955be2a9a6c7a74e08., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733340288830Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733340284444.377e6c9373fb6d955be2a9a6c7a74e08. at 1733340288830Flushing 377e6c9373fb6d955be2a9a6c7a74e08/a: creating writer at 1733340288830Flushing 377e6c9373fb6d955be2a9a6c7a74e08/a: appending metadata at 1733340288842 (+12 ms)Flushing 377e6c9373fb6d955be2a9a6c7a74e08/a: closing flushed file at 1733340288842Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4543510c: reopening flushed file at 1733340288854 (+12 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 377e6c9373fb6d955be2a9a6c7a74e08 in 30ms, sequenceid=5, compaction requested=false; wal=null at 1733340288860 (+6 ms)Cleaning up temporary data from old regions at 1733340288862 (+2 ms)Region opened successfully at 1733340288867 (+5 ms) 2024-12-04T19:24:48,870 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 377e6c9373fb6d955be2a9a6c7a74e08, disabling compactions & flushes 2024-12-04T19:24:48,870 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733340284444.377e6c9373fb6d955be2a9a6c7a74e08. 2024-12-04T19:24:48,870 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733340284444.377e6c9373fb6d955be2a9a6c7a74e08. 2024-12-04T19:24:48,871 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733340284444.377e6c9373fb6d955be2a9a6c7a74e08. after waiting 0 ms 2024-12-04T19:24:48,871 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733340284444.377e6c9373fb6d955be2a9a6c7a74e08. 2024-12-04T19:24:48,871 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733340284444.377e6c9373fb6d955be2a9a6c7a74e08. 2024-12-04T19:24:48,871 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 377e6c9373fb6d955be2a9a6c7a74e08: Waiting for close lock at 1733340288870Disabling compacts and flushes for region at 1733340288870Disabling writes for close at 1733340288871 (+1 ms)Writing region close event to WAL at 1733340288871Closed at 1733340288871 2024-12-04T19:24:48,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741883_1061 (size=175) 2024-12-04T19:24:48,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741883_1061 (size=175) 2024-12-04T19:24:48,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741883_1061 (size=175) 2024-12-04T19:24:48,876 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:24:48,876 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733340288750) 2024-12-04T19:24:48,886 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=414 (was 409) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1615926440_22 at /127.0.0.1:33450 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:33439 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1615926440_22 at /127.0.0.1:54336 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1615926440_22 at /127.0.0.1:51304 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:33439 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=180 (was 180), ProcessCount=11 (was 11), AvailableMemoryMB=15653 (was 15655) 2024-12-04T19:24:48,895 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=414, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=180, ProcessCount=11, AvailableMemoryMB=15652 2024-12-04T19:24:48,910 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:48,914 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T19:24:48,917 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9ba2734cd39f,44405,1733340272667 2024-12-04T19:24:48,919 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7becb107 2024-12-04T19:24:48,920 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T19:24:48,922 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T19:24:48,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44405 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T19:24:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44405 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-04T19:24:48,933 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T19:24:48,935 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44405 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-04T19:24:48,935 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:48,937 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T19:24:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44405 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T19:24:48,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741885_1063 (size=694) 2024-12-04T19:24:48,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741885_1063 (size=694) 2024-12-04T19:24:48,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741885_1063 (size=694) 2024-12-04T19:24:48,951 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 88a3d8cc28447f8692a6ef68159edd5f, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890 2024-12-04T19:24:48,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741886_1064 (size=77) 2024-12-04T19:24:48,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741886_1064 (size=77) 2024-12-04T19:24:48,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741886_1064 (size=77) 2024-12-04T19:24:48,961 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:48,962 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 88a3d8cc28447f8692a6ef68159edd5f, disabling compactions & flushes 2024-12-04T19:24:48,962 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:48,962 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:48,962 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. after waiting 0 ms 2024-12-04T19:24:48,962 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:48,962 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:48,962 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 88a3d8cc28447f8692a6ef68159edd5f: Waiting for close lock at 1733340288962Disabling compacts and flushes for region at 1733340288962Disabling writes for close at 1733340288962Writing region close event to WAL at 1733340288962Closed at 1733340288962 2024-12-04T19:24:48,964 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T19:24:48,969 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733340288964"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733340288964"}]},"ts":"1733340288964"} 2024-12-04T19:24:48,972 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T19:24:48,974 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T19:24:48,977 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733340288974"}]},"ts":"1733340288974"} 2024-12-04T19:24:48,981 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-04T19:24:48,982 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(204): Hosts are {9ba2734cd39f=0} racks are {/default-rack=0} 2024-12-04T19:24:48,983 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T19:24:48,983 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T19:24:48,983 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-04T19:24:48,983 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T19:24:48,983 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T19:24:48,983 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-04T19:24:48,983 INFO [PEWorker-5 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T19:24:48,984 INFO [PEWorker-5 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T19:24:48,984 INFO [PEWorker-5 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-04T19:24:48,984 DEBUG [PEWorker-5 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T19:24:48,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, ASSIGN}] 2024-12-04T19:24:48,987 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, ASSIGN 2024-12-04T19:24:48,988 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, ASSIGN; state=OFFLINE, location=9ba2734cd39f,46195,1733340273483; forceNewPlan=false, retain=false 2024-12-04T19:24:49,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44405 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T19:24:49,144 INFO [9ba2734cd39f:44405 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T19:24:49,146 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=88a3d8cc28447f8692a6ef68159edd5f, regionState=OPENING, regionLocation=9ba2734cd39f,46195,1733340273483 2024-12-04T19:24:49,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, ASSIGN because future has completed 2024-12-04T19:24:49,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88a3d8cc28447f8692a6ef68159edd5f, server=9ba2734cd39f,46195,1733340273483}] 2024-12-04T19:24:49,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44405 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T19:24:49,317 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:49,318 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 88a3d8cc28447f8692a6ef68159edd5f, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:24:49,318 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:49,319 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:49,319 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:49,319 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:49,321 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:49,323 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88a3d8cc28447f8692a6ef68159edd5f columnFamilyName cf1 2024-12-04T19:24:49,323 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:49,324 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(327): Store=88a3d8cc28447f8692a6ef68159edd5f/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:49,324 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:49,326 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88a3d8cc28447f8692a6ef68159edd5f columnFamilyName cf2 2024-12-04T19:24:49,326 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:49,327 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(327): Store=88a3d8cc28447f8692a6ef68159edd5f/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:49,327 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:49,328 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:49,329 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:49,330 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:49,330 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:49,331 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T19:24:49,332 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:49,335 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:24:49,335 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 88a3d8cc28447f8692a6ef68159edd5f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73787313, jitterRate=0.09951664507389069}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T19:24:49,335 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:49,336 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 88a3d8cc28447f8692a6ef68159edd5f: Running coprocessor pre-open hook at 1733340289319Writing region info on filesystem at 1733340289319Initializing all the Stores at 1733340289321 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340289321Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340289321Cleaning up temporary data from old regions at 1733340289330 (+9 ms)Running coprocessor post-open hooks at 1733340289335 (+5 ms)Region opened successfully at 1733340289336 (+1 ms) 2024-12-04T19:24:49,338 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., pid=6, masterSystemTime=1733340289306 2024-12-04T19:24:49,340 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:49,340 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:49,341 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=88a3d8cc28447f8692a6ef68159edd5f, regionState=OPEN, openSeqNum=2, regionLocation=9ba2734cd39f,46195,1733340273483 2024-12-04T19:24:49,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88a3d8cc28447f8692a6ef68159edd5f, server=9ba2734cd39f,46195,1733340273483 because future has completed 2024-12-04T19:24:49,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T19:24:49,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 88a3d8cc28447f8692a6ef68159edd5f, server=9ba2734cd39f,46195,1733340273483 in 192 msec 2024-12-04T19:24:49,351 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T19:24:49,352 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, ASSIGN in 364 msec 2024-12-04T19:24:49,353 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T19:24:49,353 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733340289353"}]},"ts":"1733340289353"} 2024-12-04T19:24:49,355 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-04T19:24:49,357 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T19:24:49,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 430 msec 2024-12-04T19:24:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44405 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T19:24:49,568 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-04T19:24:49,568 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-04T19:24:49,569 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T19:24:49,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-04T19:24:49,575 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T19:24:49,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-04T19:24:49,590 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,46195,1733340273483, seqNum=2] 2024-12-04T19:24:49,605 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44405 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=88a3d8cc28447f8692a6ef68159edd5f, source=9ba2734cd39f,46195,1733340273483, destination=9ba2734cd39f,42371,1733340273381, warming up region on 9ba2734cd39f,42371,1733340273381 2024-12-04T19:24:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44405 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T19:24:49,607 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44405 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=88a3d8cc28447f8692a6ef68159edd5f, source=9ba2734cd39f,46195,1733340273483, destination=9ba2734cd39f,42371,1733340273381, running balancer 2024-12-04T19:24:49,609 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33399, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T19:24:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44405 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, REOPEN/MOVE 2024-12-04T19:24:49,609 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, REOPEN/MOVE 2024-12-04T19:24:49,612 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=88a3d8cc28447f8692a6ef68159edd5f, regionState=CLOSING, regionLocation=9ba2734cd39f,46195,1733340273483 2024-12-04T19:24:49,614 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(7855): Warmup {ENCODED => 88a3d8cc28447f8692a6ef68159edd5f, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:24:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:49,615 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:49,617 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, REOPEN/MOVE because future has completed 2024-12-04T19:24:49,617 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88a3d8cc28447f8692a6ef68159edd5f columnFamilyName cf1 2024-12-04T19:24:49,618 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:49,618 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T19:24:49,618 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 88a3d8cc28447f8692a6ef68159edd5f, server=9ba2734cd39f,46195,1733340273483}] 2024-12-04T19:24:49,618 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(327): Store=88a3d8cc28447f8692a6ef68159edd5f/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:49,618 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:49,620 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88a3d8cc28447f8692a6ef68159edd5f columnFamilyName cf2 2024-12-04T19:24:49,620 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:49,620 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(327): Store=88a3d8cc28447f8692a6ef68159edd5f/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(1722): Closing 88a3d8cc28447f8692a6ef68159edd5f, disabling compactions & flushes 2024-12-04T19:24:49,621 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. after waiting 0 ms 2024-12-04T19:24:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:49,628 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] regionserver.HRegion(1676): Region close journal for 88a3d8cc28447f8692a6ef68159edd5f: Waiting for close lock at 1733340289621Disabling compacts and flushes for region at 1733340289621Disabling writes for close at 1733340289621Writing region close event to WAL at 1733340289628 (+7 ms)Closed at 1733340289628 2024-12-04T19:24:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44405 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-04T19:24:49,777 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:49,778 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T19:24:49,778 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 88a3d8cc28447f8692a6ef68159edd5f, disabling compactions & flushes 2024-12-04T19:24:49,778 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:49,778 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:49,778 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. after waiting 0 ms 2024-12-04T19:24:49,778 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:49,778 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 88a3d8cc28447f8692a6ef68159edd5f 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-04T19:24:49,800 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/.tmp/cf1/b46d8ffca42f4b319a8f11492faef599 is 35, key is r1/cf1:q/1733340289592/Put/seqid=0 2024-12-04T19:24:49,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741887_1065 (size=4783) 2024-12-04T19:24:49,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741887_1065 (size=4783) 2024-12-04T19:24:49,811 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/.tmp/cf1/b46d8ffca42f4b319a8f11492faef599 2024-12-04T19:24:49,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741887_1065 (size=4783) 2024-12-04T19:24:49,820 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/.tmp/cf1/b46d8ffca42f4b319a8f11492faef599 as hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf1/b46d8ffca42f4b319a8f11492faef599 2024-12-04T19:24:49,827 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf1/b46d8ffca42f4b319a8f11492faef599, entries=1, sequenceid=5, filesize=4.7 K 2024-12-04T19:24:49,828 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 88a3d8cc28447f8692a6ef68159edd5f in 50ms, sequenceid=5, compaction requested=false 2024-12-04T19:24:49,829 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-04T19:24:49,834 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-04T19:24:49,837 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:49,837 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 88a3d8cc28447f8692a6ef68159edd5f: Waiting for close lock at 1733340289778Running coprocessor pre-close hooks at 1733340289778Disabling compacts and flushes for region at 1733340289778Disabling writes for close at 1733340289778Obtaining lock to block concurrent updates at 1733340289778Preparing flush snapshotting stores in 88a3d8cc28447f8692a6ef68159edd5f at 1733340289778Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733340289779 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. at 1733340289779Flushing 88a3d8cc28447f8692a6ef68159edd5f/cf1: creating writer at 1733340289780 (+1 ms)Flushing 88a3d8cc28447f8692a6ef68159edd5f/cf1: appending metadata at 1733340289799 (+19 ms)Flushing 88a3d8cc28447f8692a6ef68159edd5f/cf1: closing flushed file at 1733340289800 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40047a5c: reopening flushed file at 1733340289819 (+19 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 88a3d8cc28447f8692a6ef68159edd5f in 50ms, sequenceid=5, compaction requested=false at 1733340289828 (+9 ms)Writing region close event to WAL at 1733340289830 (+2 ms)Running coprocessor post-close hooks at 1733340289835 (+5 ms)Closed at 1733340289837 (+2 ms) 2024-12-04T19:24:49,838 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 88a3d8cc28447f8692a6ef68159edd5f move to 9ba2734cd39f,42371,1733340273381 record at close sequenceid=5 2024-12-04T19:24:49,841 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:49,842 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=88a3d8cc28447f8692a6ef68159edd5f, regionState=CLOSED 2024-12-04T19:24:49,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 88a3d8cc28447f8692a6ef68159edd5f, server=9ba2734cd39f,46195,1733340273483 because future has completed 2024-12-04T19:24:49,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-04T19:24:49,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 88a3d8cc28447f8692a6ef68159edd5f, server=9ba2734cd39f,46195,1733340273483 in 228 msec 2024-12-04T19:24:49,850 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, REOPEN/MOVE; state=CLOSED, location=9ba2734cd39f,42371,1733340273381; forceNewPlan=false, retain=false 2024-12-04T19:24:50,001 INFO [9ba2734cd39f:44405 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T19:24:50,001 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=88a3d8cc28447f8692a6ef68159edd5f, regionState=OPENING, regionLocation=9ba2734cd39f,42371,1733340273381 2024-12-04T19:24:50,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, REOPEN/MOVE because future has completed 2024-12-04T19:24:50,008 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88a3d8cc28447f8692a6ef68159edd5f, server=9ba2734cd39f,42371,1733340273381}] 2024-12-04T19:24:50,170 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:50,171 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 88a3d8cc28447f8692a6ef68159edd5f, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:24:50,171 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:50,172 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:50,172 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:50,172 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:50,175 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:50,176 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88a3d8cc28447f8692a6ef68159edd5f columnFamilyName cf1 2024-12-04T19:24:50,176 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:50,184 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf1/b46d8ffca42f4b319a8f11492faef599 2024-12-04T19:24:50,184 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(327): Store=88a3d8cc28447f8692a6ef68159edd5f/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:50,184 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:50,185 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88a3d8cc28447f8692a6ef68159edd5f columnFamilyName cf2 2024-12-04T19:24:50,185 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:50,186 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(327): Store=88a3d8cc28447f8692a6ef68159edd5f/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:50,186 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:50,187 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:50,188 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:50,188 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:50,189 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:50,189 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T19:24:50,190 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:50,192 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 88a3d8cc28447f8692a6ef68159edd5f; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62709227, jitterRate=-0.06555970013141632}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T19:24:50,192 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:50,193 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 88a3d8cc28447f8692a6ef68159edd5f: Running coprocessor pre-open hook at 1733340290172Writing region info on filesystem at 1733340290172Initializing all the Stores at 1733340290174 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340290174Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340290174Cleaning up temporary data from old regions at 1733340290189 (+15 ms)Running coprocessor post-open hooks at 1733340290192 (+3 ms)Region opened successfully at 1733340290192 2024-12-04T19:24:50,194 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., pid=9, masterSystemTime=1733340290162 2024-12-04T19:24:50,197 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:50,197 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:50,198 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=88a3d8cc28447f8692a6ef68159edd5f, regionState=OPEN, openSeqNum=9, regionLocation=9ba2734cd39f,42371,1733340273381 2024-12-04T19:24:50,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88a3d8cc28447f8692a6ef68159edd5f, server=9ba2734cd39f,42371,1733340273381 because future has completed 2024-12-04T19:24:50,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-04T19:24:50,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 88a3d8cc28447f8692a6ef68159edd5f, server=9ba2734cd39f,42371,1733340273381 in 195 msec 2024-12-04T19:24:50,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, REOPEN/MOVE in 597 msec 2024-12-04T19:24:50,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T19:24:50,217 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T19:24:50,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46195 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:36028 deadline: 1733340350221, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ba2734cd39f port=42371 startCode=1733340273381. As of locationSeqNum=5. 2024-12-04T19:24:50,241 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,46195,1733340273483, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,46195,1733340273483, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ba2734cd39f port=42371 startCode=1733340273381. As of locationSeqNum=5. 2024-12-04T19:24:50,241 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,46195,1733340273483, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ba2734cd39f port=42371 startCode=1733340273381. As of locationSeqNum=5. 2024-12-04T19:24:50,242 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,46195,1733340273483, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,42371,1733340273381, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ba2734cd39f port=42371 startCode=1733340273381. As of locationSeqNum=5. 2024-12-04T19:24:50,357 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:24:50,358 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:24:50,368 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 88a3d8cc28447f8692a6ef68159edd5f 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-04T19:24:50,386 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/.tmp/cf1/0be78bb32030444cad3577254c0e3635 is 29, key is r1/cf1:/1733340290359/DeleteFamily/seqid=0 2024-12-04T19:24:50,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741888_1066 (size=4906) 2024-12-04T19:24:50,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741888_1066 (size=4906) 2024-12-04T19:24:50,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741888_1066 (size=4906) 2024-12-04T19:24:50,395 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/.tmp/cf1/0be78bb32030444cad3577254c0e3635 2024-12-04T19:24:50,401 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0be78bb32030444cad3577254c0e3635 2024-12-04T19:24:50,414 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/.tmp/cf2/35da37fd9ed545edbb3f011470008077 is 29, key is r1/cf2:/1733340290359/DeleteFamily/seqid=0 2024-12-04T19:24:50,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741889_1067 (size=4906) 2024-12-04T19:24:50,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741889_1067 (size=4906) 2024-12-04T19:24:50,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741889_1067 (size=4906) 2024-12-04T19:24:50,421 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/.tmp/cf2/35da37fd9ed545edbb3f011470008077 2024-12-04T19:24:50,426 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 35da37fd9ed545edbb3f011470008077 2024-12-04T19:24:50,427 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/.tmp/cf1/0be78bb32030444cad3577254c0e3635 as hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf1/0be78bb32030444cad3577254c0e3635 2024-12-04T19:24:50,435 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0be78bb32030444cad3577254c0e3635 2024-12-04T19:24:50,435 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf1/0be78bb32030444cad3577254c0e3635, entries=1, sequenceid=12, filesize=4.8 K 2024-12-04T19:24:50,437 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/.tmp/cf2/35da37fd9ed545edbb3f011470008077 as hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf2/35da37fd9ed545edbb3f011470008077 2024-12-04T19:24:50,444 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 35da37fd9ed545edbb3f011470008077 2024-12-04T19:24:50,444 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf2/35da37fd9ed545edbb3f011470008077, entries=1, sequenceid=12, filesize=4.8 K 2024-12-04T19:24:50,445 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 88a3d8cc28447f8692a6ef68159edd5f in 77ms, sequenceid=12, compaction requested=false 2024-12-04T19:24:50,445 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 88a3d8cc28447f8692a6ef68159edd5f: 2024-12-04T19:24:50,448 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-04T19:24:50,449 DEBUG [Time-limited test {}] regionserver.HStore(1541): 88a3d8cc28447f8692a6ef68159edd5f/cf1 is initiating major compaction (all files) 2024-12-04T19:24:50,449 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T19:24:50,449 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:50,449 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 88a3d8cc28447f8692a6ef68159edd5f/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:50,450 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf1/b46d8ffca42f4b319a8f11492faef599, hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf1/0be78bb32030444cad3577254c0e3635] into tmpdir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/.tmp, totalSize=9.5 K 2024-12-04T19:24:50,451 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting b46d8ffca42f4b319a8f11492faef599, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733340289592 2024-12-04T19:24:50,452 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 0be78bb32030444cad3577254c0e3635, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-04T19:24:50,462 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 88a3d8cc28447f8692a6ef68159edd5f#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:24:50,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741890_1068 (size=4626) 2024-12-04T19:24:50,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741890_1068 (size=4626) 2024-12-04T19:24:50,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741890_1068 (size=4626) 2024-12-04T19:24:50,479 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/.tmp/cf1/e0a1a2e721fe4459b572121da574a18c as hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf1/e0a1a2e721fe4459b572121da574a18c 2024-12-04T19:24:50,493 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 88a3d8cc28447f8692a6ef68159edd5f/cf1 of 88a3d8cc28447f8692a6ef68159edd5f into e0a1a2e721fe4459b572121da574a18c(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:24:50,493 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 88a3d8cc28447f8692a6ef68159edd5f: 2024-12-04T19:24:50,493 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-04T19:24:50,493 DEBUG [Time-limited test {}] regionserver.HStore(1541): 88a3d8cc28447f8692a6ef68159edd5f/cf2 is initiating major compaction (all files) 2024-12-04T19:24:50,493 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T19:24:50,493 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T19:24:50,493 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 88a3d8cc28447f8692a6ef68159edd5f/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:50,494 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf2/35da37fd9ed545edbb3f011470008077] into tmpdir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/.tmp, totalSize=4.8 K 2024-12-04T19:24:50,494 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 35da37fd9ed545edbb3f011470008077, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-04T19:24:50,500 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 88a3d8cc28447f8692a6ef68159edd5f#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T19:24:50,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741891_1069 (size=4592) 2024-12-04T19:24:50,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741891_1069 (size=4592) 2024-12-04T19:24:50,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741891_1069 (size=4592) 2024-12-04T19:24:50,518 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/.tmp/cf2/9032ae5e5a524ac9ab93d3e53caf3798 as hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf2/9032ae5e5a524ac9ab93d3e53caf3798 2024-12-04T19:24:50,525 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 88a3d8cc28447f8692a6ef68159edd5f/cf2 of 88a3d8cc28447f8692a6ef68159edd5f into 9032ae5e5a524ac9ab93d3e53caf3798(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:24:50,525 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 88a3d8cc28447f8692a6ef68159edd5f: 2024-12-04T19:24:50,529 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44405 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=88a3d8cc28447f8692a6ef68159edd5f, source=9ba2734cd39f,42371,1733340273381, destination=9ba2734cd39f,46195,1733340273483, warming up region on 9ba2734cd39f,46195,1733340273483 2024-12-04T19:24:50,529 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44405 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=88a3d8cc28447f8692a6ef68159edd5f, source=9ba2734cd39f,42371,1733340273381, destination=9ba2734cd39f,46195,1733340273483, running balancer 2024-12-04T19:24:50,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44405 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, REOPEN/MOVE 2024-12-04T19:24:50,530 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, REOPEN/MOVE 2024-12-04T19:24:50,532 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=88a3d8cc28447f8692a6ef68159edd5f, regionState=CLOSING, regionLocation=9ba2734cd39f,42371,1733340273381 2024-12-04T19:24:50,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46195 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:50,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46195 {}] regionserver.HRegion(7855): Warmup {ENCODED => 88a3d8cc28447f8692a6ef68159edd5f, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:24:50,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46195 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:50,534 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:50,535 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88a3d8cc28447f8692a6ef68159edd5f columnFamilyName cf1 2024-12-04T19:24:50,535 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:50,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, REOPEN/MOVE because future has completed 2024-12-04T19:24:50,535 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T19:24:50,535 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 88a3d8cc28447f8692a6ef68159edd5f, server=9ba2734cd39f,42371,1733340273381}] 2024-12-04T19:24:50,542 INFO [StoreFileOpener-88a3d8cc28447f8692a6ef68159edd5f-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0be78bb32030444cad3577254c0e3635 2024-12-04T19:24:50,543 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf1/0be78bb32030444cad3577254c0e3635 2024-12-04T19:24:50,547 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf1/b46d8ffca42f4b319a8f11492faef599 2024-12-04T19:24:50,552 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf1/e0a1a2e721fe4459b572121da574a18c 2024-12-04T19:24:50,552 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(327): Store=88a3d8cc28447f8692a6ef68159edd5f/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:50,553 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:50,554 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88a3d8cc28447f8692a6ef68159edd5f columnFamilyName cf2 2024-12-04T19:24:50,554 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:50,560 INFO [StoreFileOpener-88a3d8cc28447f8692a6ef68159edd5f-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 35da37fd9ed545edbb3f011470008077 2024-12-04T19:24:50,561 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf2/35da37fd9ed545edbb3f011470008077 2024-12-04T19:24:50,566 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf2/9032ae5e5a524ac9ab93d3e53caf3798 2024-12-04T19:24:50,566 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(327): Store=88a3d8cc28447f8692a6ef68159edd5f/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:50,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46195 {}] regionserver.HRegion(1722): Closing 88a3d8cc28447f8692a6ef68159edd5f, disabling compactions & flushes 2024-12-04T19:24:50,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46195 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:50,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46195 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:50,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46195 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. after waiting 0 ms 2024-12-04T19:24:50,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46195 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:50,567 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46195 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:50,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46195 {}] regionserver.HRegion(1676): Region close journal for 88a3d8cc28447f8692a6ef68159edd5f: Waiting for close lock at 1733340290566Disabling compacts and flushes for region at 1733340290566Disabling writes for close at 1733340290566Writing region close event to WAL at 1733340290567 (+1 ms)Closed at 1733340290567 2024-12-04T19:24:50,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44405 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-04T19:24:50,690 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:50,690 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T19:24:50,691 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 88a3d8cc28447f8692a6ef68159edd5f, disabling compactions & flushes 2024-12-04T19:24:50,691 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:50,691 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:50,691 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. after waiting 0 ms 2024-12-04T19:24:50,691 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:50,692 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf1/b46d8ffca42f4b319a8f11492faef599, hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf1/0be78bb32030444cad3577254c0e3635] to archive 2024-12-04T19:24:50,696 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:24:50,702 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf1/b46d8ffca42f4b319a8f11492faef599 to hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf1/b46d8ffca42f4b319a8f11492faef599 2024-12-04T19:24:50,704 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf1/0be78bb32030444cad3577254c0e3635 to hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf1/0be78bb32030444cad3577254c0e3635 2024-12-04T19:24:50,717 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf2/35da37fd9ed545edbb3f011470008077] to archive 2024-12-04T19:24:50,718 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:24:50,720 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf2/35da37fd9ed545edbb3f011470008077 to hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf2/35da37fd9ed545edbb3f011470008077 2024-12-04T19:24:50,725 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-04T19:24:50,726 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:50,726 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 88a3d8cc28447f8692a6ef68159edd5f: Waiting for close lock at 1733340290691Running coprocessor pre-close hooks at 1733340290691Disabling compacts and flushes for region at 1733340290691Disabling writes for close at 1733340290691Writing region close event to WAL at 1733340290721 (+30 ms)Running coprocessor post-close hooks at 1733340290726 (+5 ms)Closed at 1733340290726 2024-12-04T19:24:50,726 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 88a3d8cc28447f8692a6ef68159edd5f move to 9ba2734cd39f,46195,1733340273483 record at close sequenceid=12 2024-12-04T19:24:50,728 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:50,729 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=88a3d8cc28447f8692a6ef68159edd5f, regionState=CLOSED 2024-12-04T19:24:50,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 88a3d8cc28447f8692a6ef68159edd5f, server=9ba2734cd39f,42371,1733340273381 because future has completed 2024-12-04T19:24:50,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-04T19:24:50,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 88a3d8cc28447f8692a6ef68159edd5f, server=9ba2734cd39f,42371,1733340273381 in 198 msec 2024-12-04T19:24:50,737 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, REOPEN/MOVE; state=CLOSED, location=9ba2734cd39f,46195,1733340273483; forceNewPlan=false, retain=false 2024-12-04T19:24:50,887 INFO [9ba2734cd39f:44405 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T19:24:50,888 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=88a3d8cc28447f8692a6ef68159edd5f, regionState=OPENING, regionLocation=9ba2734cd39f,46195,1733340273483 2024-12-04T19:24:50,894 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, REOPEN/MOVE because future has completed 2024-12-04T19:24:50,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88a3d8cc28447f8692a6ef68159edd5f, server=9ba2734cd39f,46195,1733340273483}] 2024-12-04T19:24:51,052 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:51,053 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 88a3d8cc28447f8692a6ef68159edd5f, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:24:51,053 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:51,053 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:51,053 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:51,053 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:51,055 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:51,055 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88a3d8cc28447f8692a6ef68159edd5f columnFamilyName cf1 2024-12-04T19:24:51,055 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:51,061 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf1/e0a1a2e721fe4459b572121da574a18c 2024-12-04T19:24:51,061 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(327): Store=88a3d8cc28447f8692a6ef68159edd5f/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:51,061 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:51,062 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88a3d8cc28447f8692a6ef68159edd5f columnFamilyName cf2 2024-12-04T19:24:51,062 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:51,069 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf2/9032ae5e5a524ac9ab93d3e53caf3798 2024-12-04T19:24:51,069 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(327): Store=88a3d8cc28447f8692a6ef68159edd5f/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:51,069 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:51,070 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:51,071 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:51,072 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:51,072 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:51,073 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T19:24:51,074 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:51,075 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 88a3d8cc28447f8692a6ef68159edd5f; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71738368, jitterRate=0.0689849853515625}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T19:24:51,075 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:51,076 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 88a3d8cc28447f8692a6ef68159edd5f: Running coprocessor pre-open hook at 1733340291053Writing region info on filesystem at 1733340291053Initializing all the Stores at 1733340291054 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340291054Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340291054Cleaning up temporary data from old regions at 1733340291072 (+18 ms)Running coprocessor post-open hooks at 1733340291075 (+3 ms)Region opened successfully at 1733340291076 (+1 ms) 2024-12-04T19:24:51,077 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., pid=12, masterSystemTime=1733340291049 2024-12-04T19:24:51,079 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:51,080 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:51,080 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=88a3d8cc28447f8692a6ef68159edd5f, regionState=OPEN, openSeqNum=18, regionLocation=9ba2734cd39f,46195,1733340273483 2024-12-04T19:24:51,083 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88a3d8cc28447f8692a6ef68159edd5f, server=9ba2734cd39f,46195,1733340273483 because future has completed 2024-12-04T19:24:51,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-04T19:24:51,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 88a3d8cc28447f8692a6ef68159edd5f, server=9ba2734cd39f,46195,1733340273483 in 190 msec 2024-12-04T19:24:51,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, REOPEN/MOVE in 558 msec 2024-12-04T19:24:51,132 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T19:24:51,134 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T19:24:51,136 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 9ba2734cd39f,46195,1733340273483: testing ***** 2024-12-04T19:24:51,136 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-04T19:24:51,138 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-04T19:24:51,139 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-04T19:24:51,141 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-04T19:24:51,142 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-04T19:24:51,148 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 478279216 }, "NonHeapMemoryUsage": { "committed": 171376640, "init": 7667712, "max": -1, "used": 168755056 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "9ba2734cd39f", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 9, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 4, "ProcessCallTime_75th_percentile": 6, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 8, "ProcessCallTime_98th_percentile": 8, "ProcessCallTime_99th_percentile": 8, "ProcessCallTime_99.9th_percentile": 8, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 9, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 8, "TotalCallTime_95th_percentile": 8, "TotalCallTime_98th_percentile": 8, "TotalCallTime_99th_percentile": 8, "TotalCallTime_99.9th_percentile": 8, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "9ba2734cd39f", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:61732", "tag.serverName": "9ba2734cd39f,46195,1733340273483", "tag.clusterId": "ad1ba118-c3ab-45fe-b5d1-9fce4b581c96", "tag.Context": "regionserver", "tag.Hostname": "9ba2734cd39f", "regionCount": 1, "storeCount": 4, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733340273483, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 0.4, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 199680, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 1, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 4, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 4, "ScanTime_min": 0, "ScanTime_max": 2, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 1, "ScanTime_95th_percentile": 1, "ScanTime_98th_percentile": 1, "ScanTime_99th_percentile": 1, "ScanTime_99.9th_percentile": 1, "ScanTime_TimeRangeCount_0-1": 4, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 2, "Put_min": 2, "Put_max": 7, "Put_mean": 4, "Put_25th_percentile": 3, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 2, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 1, "Get_min": 2, "Get_max": 2, "Get_mean": 2, "Get_25th_percentile": 2, "Get_median": 2, "Get_75th_percentile": 2, "Get_90th_percentile": 2, "Get_95th_percentile": 2, "Get_98th_percentile": 2, "Get_99th_percentile": 2, "Get_99.9th_percentile": 2, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 4, "ScanSize_min": 0, "ScanSize_max": 144, "ScanSize_mean": 72, "ScanSize_25th_percentile": 36, "ScanSize_median": 72, "ScanSize_75th_percentile": 108, "ScanSize_90th_percentile": 129, "ScanSize_95th_percentile": 136, "ScanSize_98th_percentile": 141, "ScanSize_99th_percentile": 142, "ScanSize_99.9th_percentile": 143, "ScanSize_SizeRangeCount_0-10": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-04T19:24:51,150 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44405 {}] master.MasterRpcServices(700): 9ba2734cd39f,46195,1733340273483 reported a fatal error: ***** ABORTING region server 9ba2734cd39f,46195,1733340273483: testing ***** 2024-12-04T19:24:51,152 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9ba2734cd39f,46195,1733340273483' ***** 2024-12-04T19:24:51,152 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-04T19:24:51,153 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T19:24:51,153 INFO [RS:1;9ba2734cd39f:46195 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-04T19:24:51,153 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T19:24:51,153 INFO [RS:1;9ba2734cd39f:46195 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-04T19:24:51,153 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.HRegionServer(3091): Received CLOSE for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:51,154 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.HRegionServer(956): aborting server 9ba2734cd39f,46195,1733340273483 2024-12-04T19:24:51,154 INFO [RS:1;9ba2734cd39f:46195 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T19:24:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:42702 deadline: 1733340351154, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ba2734cd39f port=46195 startCode=1733340273483. As of locationSeqNum=12. 2024-12-04T19:24:51,154 INFO [RS:1;9ba2734cd39f:46195 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;9ba2734cd39f:46195. 2024-12-04T19:24:51,154 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 88a3d8cc28447f8692a6ef68159edd5f, disabling compactions & flushes 2024-12-04T19:24:51,154 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:51,154 DEBUG [RS:1;9ba2734cd39f:46195 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T19:24:51,154 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:51,154 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. after waiting 0 ms 2024-12-04T19:24:51,154 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:51,155 DEBUG [RS:1;9ba2734cd39f:46195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:24:51,155 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,42371,1733340273381, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,42371,1733340273381, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ba2734cd39f port=46195 startCode=1733340273483. As of locationSeqNum=12. 2024-12-04T19:24:51,155 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,42371,1733340273381, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ba2734cd39f port=46195 startCode=1733340273483. As of locationSeqNum=12. 2024-12-04T19:24:51,155 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,42371,1733340273381, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,46195,1733340273483, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=9ba2734cd39f port=46195 startCode=1733340273483. As of locationSeqNum=12. 2024-12-04T19:24:51,155 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T19:24:51,155 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T19:24:51,155 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T19:24:51,155 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T19:24:51,156 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-04T19:24:51,156 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T19:24:51,156 DEBUG [RS:1;9ba2734cd39f:46195 {}] regionserver.HRegionServer(1325): Online Regions={88a3d8cc28447f8692a6ef68159edd5f=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., 1588230740=hbase:meta,,1.1588230740} 2024-12-04T19:24:51,157 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T19:24:51,157 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T19:24:51,157 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T19:24:51,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9ba2734cd39f,46195,1733340273483 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:24:51,157 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T19:24:51,157 DEBUG [RS:1;9ba2734cd39f:46195 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:51,157 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:51,157 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 88a3d8cc28447f8692a6ef68159edd5f: Waiting for close lock at 1733340291154Running coprocessor pre-close hooks at 1733340291154Disabling compacts and flushes for region at 1733340291154Disabling writes for close at 1733340291154Writing region close event to WAL at 1733340291157 (+3 ms)Running coprocessor post-close hooks at 1733340291157Closed at 1733340291157 2024-12-04T19:24:51,157 ERROR [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-12-04T19:24:51,157 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:51,157 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T19:24:51,158 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T19:24:51,158 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733340291156Running coprocessor pre-close hooks at 1733340291156Disabling compacts and flushes for region at 1733340291156Disabling writes for close at 1733340291157 (+1 ms)Writing region close event to WAL at 1733340291157Running coprocessor post-close hooks at 1733340291157Closed at 1733340291158 (+1 ms) 2024-12-04T19:24:51,158 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9ba2734cd39f,46195,1733340273483 aborting 2024-12-04T19:24:51,158 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T19:24:51,158 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9ba2734cd39f,46195,1733340273483 aborting 2024-12-04T19:24:51,158 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1 from cache 2024-12-04T19:24:51,199 INFO [regionserver/9ba2734cd39f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T19:24:51,267 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:24:51,268 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1] 2024-12-04T19:24:51,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9ba2734cd39f,46195,1733340273483 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:24:51,269 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9ba2734cd39f,46195,1733340273483 aborting 2024-12-04T19:24:51,269 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 9ba2734cd39f,46195,1733340273483 aborting 2024-12-04T19:24:51,269 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1 from cache 2024-12-04T19:24:51,357 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.HRegionServer(976): stopping server 9ba2734cd39f,46195,1733340273483; all regions closed. 2024-12-04T19:24:51,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741836_1012 (size=7323) 2024-12-04T19:24:51,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741836_1012 (size=7323) 2024-12-04T19:24:51,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741836_1012 (size=7323) 2024-12-04T19:24:51,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741833_1009 (size=2288) 2024-12-04T19:24:51,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741833_1009 (size=2288) 2024-12-04T19:24:51,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741833_1009 (size=2288) 2024-12-04T19:24:51,370 DEBUG [RS:1;9ba2734cd39f:46195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:24:51,370 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T19:24:51,370 INFO [RS:1;9ba2734cd39f:46195 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T19:24:51,370 INFO [RS:1;9ba2734cd39f:46195 {}] hbase.ChoreService(370): Chore service for: regionserver/9ba2734cd39f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T19:24:51,370 INFO [RS:1;9ba2734cd39f:46195 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T19:24:51,370 INFO [regionserver/9ba2734cd39f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T19:24:51,371 INFO [RS:1;9ba2734cd39f:46195 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46195 2024-12-04T19:24:51,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46195-0x101a1cd90bd0002, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9ba2734cd39f,46195,1733340273483 2024-12-04T19:24:51,395 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T19:24:51,395 INFO [RS:1;9ba2734cd39f:46195 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T19:24:51,397 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9ba2734cd39f,46195,1733340273483] 2024-12-04T19:24:51,416 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9ba2734cd39f,46195,1733340273483 already deleted, retry=false 2024-12-04T19:24:51,417 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 9ba2734cd39f,46195,1733340273483 on 9ba2734cd39f,44405,1733340272667 2024-12-04T19:24:51,428 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 9ba2734cd39f,46195,1733340273483, splitWal=true, meta=true 2024-12-04T19:24:51,429 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 9ba2734cd39f,46195,1733340273483 (carryingMeta=true) 9ba2734cd39f,46195,1733340273483/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@ed2a3fb[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-04T19:24:51,430 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 9ba2734cd39f,46195,1733340273483, splitWal=true, meta=true 2024-12-04T19:24:51,431 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 9ba2734cd39f,46195,1733340273483, splitWal=true, meta=true, isMeta: true 2024-12-04T19:24:51,432 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting 2024-12-04T19:24:51,433 INFO [PEWorker-1 {}] master.SplitWALManager(105): 9ba2734cd39f,46195,1733340273483 WAL count=1, meta=true 2024-12-04T19:24:51,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta}] 2024-12-04T19:24:51,439 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=9ba2734cd39f,42371,1733340273381 2024-12-04T19:24:51,441 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta, worker=9ba2734cd39f,42371,1733340273381}] 2024-12-04T19:24:51,476 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:24:51,477 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1] 2024-12-04T19:24:51,479 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 9ba2734cd39f:46195 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 9ba2734cd39f/172.17.0.2:46195 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:24:51,479 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1, error=java.net.ConnectException: Call to address=9ba2734cd39f:46195 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 9ba2734cd39f/172.17.0.2:46195 2024-12-04T19:24:51,479 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1 is java.net.ConnectException: Connection refused 2024-12-04T19:24:51,479 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1 from cache 2024-12-04T19:24:51,480 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 9ba2734cd39f:46195 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 9ba2734cd39f/172.17.0.2:46195 2024-12-04T19:24:51,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46195-0x101a1cd90bd0002, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T19:24:51,506 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46195-0x101a1cd90bd0002, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T19:24:51,507 INFO [RS:1;9ba2734cd39f:46195 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T19:24:51,507 INFO [RS:1;9ba2734cd39f:46195 {}] regionserver.HRegionServer(1031): Exiting; stopping=9ba2734cd39f,46195,1733340273483; zookeeper connection closed. 2024-12-04T19:24:51,508 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@64de47da {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@64de47da 2024-12-04T19:24:51,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-04T19:24:51,614 INFO [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta, size=7.2 K (7323bytes) 2024-12-04T19:24:51,614 INFO [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta 2024-12-04T19:24:51,615 INFO [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta after 1ms 2024-12-04T19:24:51,636 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:51,636 INFO [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta took 22ms 2024-12-04T19:24:51,642 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-04T19:24:51,642 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta so closing down 2024-12-04T19:24:51,642 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:24:51,644 INFO [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta.temp 2024-12-04T19:24:51,666 INFO [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta.temp 2024-12-04T19:24:51,667 INFO [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:24:51,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741892_1070 (size=7071) 2024-12-04T19:24:51,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741892_1070 (size=7071) 2024-12-04T19:24:51,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741892_1070 (size=7071) 2024-12-04T19:24:51,675 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-04T19:24:51,677 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta.temp to hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-04T19:24:51,677 INFO [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 41 ms; skipped=1; WAL=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false 2024-12-04T19:24:51,677 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta, journal: Splitting hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta, size=7.2 K (7323bytes) at 1733340291614Finishing writing output for hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta so closing down at 1733340291642 (+28 ms)Creating recovered edits writer path=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta.temp at 1733340291666 (+24 ms)3 split writer threads finished at 1733340291667 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733340291676 (+9 ms)Rename recovered edits hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta.temp to hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1733340291677 (+1 ms)Processed 16 edits across 1 Regions in 41 ms; skipped=1; WAL=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false at 1733340291677 2024-12-04T19:24:51,677 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta 2024-12-04T19:24:51,679 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-04T19:24:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44405 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-04T19:24:51,685 INFO [PEWorker-3 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta to hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/oldWALs 2024-12-04T19:24:51,689 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-04T19:24:51,689 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta, worker=9ba2734cd39f,42371,1733340273381 in 245 msec 2024-12-04T19:24:51,690 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=9ba2734cd39f,42371,1733340273381 2024-12-04T19:24:51,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-04T19:24:51,694 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 9ba2734cd39f%2C46195%2C1733340273483.meta.1733340275616.meta, worker=9ba2734cd39f,42371,1733340273381 in 257 msec 2024-12-04T19:24:51,695 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting dir is empty, no logs to split. 2024-12-04T19:24:51,695 INFO [PEWorker-1 {}] master.SplitWALManager(105): 9ba2734cd39f,46195,1733340273483 WAL count=0, meta=true 2024-12-04T19:24:51,695 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 9ba2734cd39f,46195,1733340273483 WAL splitting is done? wals=0, meta=true 2024-12-04T19:24:51,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T19:24:51,700 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T19:24:51,701 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-04T19:24:51,786 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:24:51,787 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1] 2024-12-04T19:24:51,788 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 9ba2734cd39f:46195 this server is in the failed servers list 2024-12-04T19:24:51,788 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=9ba2734cd39f:46195 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9ba2734cd39f:46195 2024-12-04T19:24:51,788 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9ba2734cd39f:46195 2024-12-04T19:24:51,788 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1 from cache 2024-12-04T19:24:51,851 DEBUG [9ba2734cd39f:44405 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-04T19:24:51,851 DEBUG [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(204): Hosts are {9ba2734cd39f=0} racks are {/default-rack=0} 2024-12-04T19:24:51,851 DEBUG [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T19:24:51,851 DEBUG [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T19:24:51,851 DEBUG [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T19:24:51,851 DEBUG [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T19:24:51,851 INFO [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T19:24:51,852 INFO [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T19:24:51,852 DEBUG [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T19:24:51,852 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9ba2734cd39f,42371,1733340273381 2024-12-04T19:24:51,853 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9ba2734cd39f,42371,1733340273381, state=OPENING 2024-12-04T19:24:51,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T19:24:51,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T19:24:51,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T19:24:51,901 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T19:24:51,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9ba2734cd39f,42371,1733340273381}] 2024-12-04T19:24:51,901 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:24:51,901 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:24:51,901 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:24:52,062 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T19:24:52,062 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:52,062 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-04T19:24:52,064 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9ba2734cd39f%2C42371%2C1733340273381.meta, suffix=.meta, logDir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,42371,1733340273381, archiveDir=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/oldWALs, maxLogs=32 2024-12-04T19:24:52,079 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,42371,1733340273381/9ba2734cd39f%2C42371%2C1733340273381.meta.1733340292065.meta, exclude list is [], retry=0 2024-12-04T19:24:52,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:52,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:52,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:52,105 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,42371,1733340273381/9ba2734cd39f%2C42371%2C1733340273381.meta.1733340292065.meta 2024-12-04T19:24:52,106 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:33421:33421)] 2024-12-04T19:24:52,106 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:24:52,106 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T19:24:52,106 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T19:24:52,107 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T19:24:52,107 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T19:24:52,107 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:52,107 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T19:24:52,107 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T19:24:52,108 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T19:24:52,109 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T19:24:52,109 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:52,110 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:24:52,110 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T19:24:52,111 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T19:24:52,111 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:52,111 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:24:52,111 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T19:24:52,112 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T19:24:52,112 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:52,112 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:24:52,113 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T19:24:52,113 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T19:24:52,113 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:52,114 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T19:24:52,114 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T19:24:52,115 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740 2024-12-04T19:24:52,116 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740 2024-12-04T19:24:52,117 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-04T19:24:52,139 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:52,141 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-04T19:24:52,141 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-12-04T19:24:52,157 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/.tmp/info/74fb0ea508fe4729a02fcae649080f7f is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f./info:regioninfo/1733340291080/Put/seqid=0 2024-12-04T19:24:52,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741894_1072 (size=11177) 2024-12-04T19:24:52,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741894_1072 (size=11177) 2024-12-04T19:24:52,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741894_1072 (size=11177) 2024-12-04T19:24:52,164 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/.tmp/info/74fb0ea508fe4729a02fcae649080f7f 2024-12-04T19:24:52,185 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/.tmp/ns/7cd0d33ba2ec4366b6569be643984c4d is 43, key is default/ns:d/1733340275836/Put/seqid=0 2024-12-04T19:24:52,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741895_1073 (size=5153) 2024-12-04T19:24:52,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741895_1073 (size=5153) 2024-12-04T19:24:52,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741895_1073 (size=5153) 2024-12-04T19:24:52,192 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/.tmp/ns/7cd0d33ba2ec4366b6569be643984c4d 2024-12-04T19:24:52,211 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/.tmp/table/19e8f7991596430196ae57da2d6e5069 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733340289353/Put/seqid=0 2024-12-04T19:24:52,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741896_1074 (size=5431) 2024-12-04T19:24:52,218 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/.tmp/table/19e8f7991596430196ae57da2d6e5069 2024-12-04T19:24:52,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741896_1074 (size=5431) 2024-12-04T19:24:52,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741896_1074 (size=5431) 2024-12-04T19:24:52,224 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/.tmp/info/74fb0ea508fe4729a02fcae649080f7f as hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/info/74fb0ea508fe4729a02fcae649080f7f 2024-12-04T19:24:52,230 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/info/74fb0ea508fe4729a02fcae649080f7f, entries=36, sequenceid=18, filesize=10.9 K 2024-12-04T19:24:52,231 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/.tmp/ns/7cd0d33ba2ec4366b6569be643984c4d as hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/ns/7cd0d33ba2ec4366b6569be643984c4d 2024-12-04T19:24:52,237 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/ns/7cd0d33ba2ec4366b6569be643984c4d, entries=2, sequenceid=18, filesize=5.0 K 2024-12-04T19:24:52,238 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/.tmp/table/19e8f7991596430196ae57da2d6e5069 as hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/table/19e8f7991596430196ae57da2d6e5069 2024-12-04T19:24:52,243 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/table/19e8f7991596430196ae57da2d6e5069, entries=2, sequenceid=18, filesize=5.3 K 2024-12-04T19:24:52,244 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 102ms, sequenceid=18, compaction requested=false; wal=null 2024-12-04T19:24:52,244 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-04T19:24:52,246 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T19:24:52,246 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T19:24:52,246 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T19:24:52,248 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T19:24:52,250 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-04T19:24:52,251 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69172729, jitterRate=0.030753985047340393}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T19:24:52,252 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T19:24:52,252 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733340292107Writing region info on filesystem at 1733340292107Initializing all the Stores at 1733340292108 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733340292108Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733340292108Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340292108Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733340292108Obtaining lock to block concurrent updates at 1733340292141 (+33 ms)Preparing flush snapshotting stores in 1588230740 at 1733340292141Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1733340292141Flushing stores of hbase:meta,,1.1588230740 at 1733340292141Flushing 1588230740/info: creating writer at 1733340292142 (+1 ms)Flushing 1588230740/info: appending metadata at 1733340292157 (+15 ms)Flushing 1588230740/info: closing flushed file at 1733340292157Flushing 1588230740/ns: creating writer at 1733340292171 (+14 ms)Flushing 1588230740/ns: appending metadata at 1733340292185 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733340292185Flushing 1588230740/table: creating writer at 1733340292197 (+12 ms)Flushing 1588230740/table: appending metadata at 1733340292211 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733340292211Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@210a6c99: reopening flushed file at 1733340292223 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@799ebb53: reopening flushed file at 1733340292230 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b8140b1: reopening flushed file at 1733340292237 (+7 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 102ms, sequenceid=18, compaction requested=false; wal=null at 1733340292244 (+7 ms)Cleaning up temporary data from old regions at 1733340292246 (+2 ms)Running coprocessor post-open hooks at 1733340292252 (+6 ms)Region opened successfully at 1733340292252 2024-12-04T19:24:52,253 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1733340292057 2024-12-04T19:24:52,255 DEBUG [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T19:24:52,255 INFO [RS_OPEN_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T19:24:52,256 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=9ba2734cd39f,42371,1733340273381 2024-12-04T19:24:52,257 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9ba2734cd39f,42371,1733340273381, state=OPEN 2024-12-04T19:24:52,261 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44405 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=9ba2734cd39f,42371,1733340273381, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-04T19:24:52,296 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:24:52,298 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,42371,1733340273381, seqNum=-1] 2024-12-04T19:24:52,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T19:24:52,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T19:24:52,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T19:24:52,307 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:24:52,307 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:24:52,307 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T19:24:52,307 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9ba2734cd39f,42371,1733340273381 2024-12-04T19:24:52,310 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-04T19:24:52,310 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9ba2734cd39f,42371,1733340273381 in 406 msec 2024-12-04T19:24:52,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-04T19:24:52,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 612 msec 2024-12-04T19:24:52,312 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 9ba2734cd39f,46195,1733340273483 had 2 regions 2024-12-04T19:24:52,313 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 9ba2734cd39f,46195,1733340273483, splitWal=true, meta=true, isMeta: false 2024-12-04T19:24:52,315 INFO [PEWorker-2 {}] master.SplitWALManager(105): 9ba2734cd39f,46195,1733340273483 WAL count=1, meta=false 2024-12-04T19:24:52,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 9ba2734cd39f%2C46195%2C1733340273483.1733340275242}] 2024-12-04T19:24:52,317 DEBUG [PEWorker-5 {}] master.SplitWALManager(158): Acquired split WAL worker=9ba2734cd39f,38513,1733340273533 2024-12-04T19:24:52,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 9ba2734cd39f%2C46195%2C1733340273483.1733340275242, worker=9ba2734cd39f,38513,1733340273533}] 2024-12-04T19:24:52,417 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 9ba2734cd39f:46195 this server is in the failed servers list 2024-12-04T19:24:52,417 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,46195,1733340273483, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,46195,1733340273483, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=9ba2734cd39f:46195 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9ba2734cd39f:46195 2024-12-04T19:24:52,417 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,46195,1733340273483, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9ba2734cd39f:46195 2024-12-04T19:24:52,417 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,46195,1733340273483, seqNum=12 from cache 2024-12-04T19:24:52,470 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T19:24:52,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51121, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T19:24:52,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38513 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-12-04T19:24:52,492 INFO [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.1733340275242, size=2.2 K (2288bytes) 2024-12-04T19:24:52,492 INFO [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.1733340275242 2024-12-04T19:24:52,493 INFO [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.1733340275242 after 1ms 2024-12-04T19:24:52,515 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:52,515 INFO [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.1733340275242 took 23ms 2024-12-04T19:24:52,518 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 88a3d8cc28447f8692a6ef68159edd5f: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-04T19:24:52,518 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.1733340275242 so closing down 2024-12-04T19:24:52,518 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:24:52,518 INFO [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:24:52,518 INFO [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 2 ms; skipped=6; WAL=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.1733340275242, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-12-04T19:24:52,518 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.1733340275242, journal: Splitting hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.1733340275242, size=2.2 K (2288bytes) at 1733340292492Finishing writing output for hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.1733340275242 so closing down at 1733340292518 (+26 ms)3 split writer threads finished at 1733340292518Processed 6 edits across 0 Regions in 2 ms; skipped=6; WAL=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.1733340275242, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1733340292518 2024-12-04T19:24:52,518 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.1733340275242 2024-12-04T19:24:52,518 DEBUG [RS_LOG_REPLAY_OPS-regionserver/9ba2734cd39f:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-12-04T19:24:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44405 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-12-04T19:24:52,522 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting/9ba2734cd39f%2C46195%2C1733340273483.1733340275242 to hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/oldWALs 2024-12-04T19:24:52,525 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-12-04T19:24:52,525 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 9ba2734cd39f%2C46195%2C1733340273483.1733340275242, worker=9ba2734cd39f,38513,1733340273533 in 205 msec 2024-12-04T19:24:52,526 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=9ba2734cd39f,38513,1733340273533 2024-12-04T19:24:52,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-12-04T19:24:52,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 9ba2734cd39f%2C46195%2C1733340273483.1733340275242, worker=9ba2734cd39f,38513,1733340273533 in 212 msec 2024-12-04T19:24:52,530 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,46195,1733340273483-splitting dir is empty, no logs to split. 2024-12-04T19:24:52,530 INFO [PEWorker-2 {}] master.SplitWALManager(105): 9ba2734cd39f,46195,1733340273483 WAL count=0, meta=false 2024-12-04T19:24:52,530 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 9ba2734cd39f,46195,1733340273483 WAL splitting is done? wals=0, meta=false 2024-12-04T19:24:52,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, ASSIGN}] 2024-12-04T19:24:52,533 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, ASSIGN 2024-12-04T19:24:52,534 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-04T19:24:52,635 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,46195,1733340273483, seqNum=18] 2024-12-04T19:24:52,635 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 9ba2734cd39f:46195 this server is in the failed servers list 2024-12-04T19:24:52,636 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,46195,1733340273483, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,46195,1733340273483, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=9ba2734cd39f:46195 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9ba2734cd39f:46195 2024-12-04T19:24:52,636 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,46195,1733340273483, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9ba2734cd39f:46195 2024-12-04T19:24:52,636 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,46195,1733340273483, seqNum=18 from cache 2024-12-04T19:24:52,685 DEBUG [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(204): Hosts are {9ba2734cd39f=0} racks are {/default-rack=0} 2024-12-04T19:24:52,685 DEBUG [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T19:24:52,685 DEBUG [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T19:24:52,685 DEBUG [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T19:24:52,685 DEBUG [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T19:24:52,685 INFO [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T19:24:52,685 INFO [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T19:24:52,685 DEBUG [9ba2734cd39f:44405 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T19:24:52,685 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=88a3d8cc28447f8692a6ef68159edd5f, regionState=OPENING, regionLocation=9ba2734cd39f,42371,1733340273381 2024-12-04T19:24:52,686 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 9ba2734cd39f:46195 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9ba2734cd39f/172.17.0.2:46195 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:24:52,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1, error=java.net.ConnectException: Call to address=9ba2734cd39f:46195 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9ba2734cd39f/172.17.0.2:46195 2024-12-04T19:24:52,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-04T19:24:52,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,46195,1733340273483, seqNum=-1 from cache 2024-12-04T19:24:52,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 9ba2734cd39f:46195 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 9ba2734cd39f/172.17.0.2:46195 2024-12-04T19:24:52,797 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T19:24:52,797 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9ba2734cd39f,42371,1733340273381, seqNum=-1] 2024-12-04T19:24:52,797 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T19:24:52,799 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39957, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T19:24:52,802 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, ASSIGN because future has completed 2024-12-04T19:24:52,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88a3d8cc28447f8692a6ef68159edd5f, server=9ba2734cd39f,42371,1733340273381}] 2024-12-04T19:24:52,958 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,46195,1733340273483, seqNum=18] 2024-12-04T19:24:52,958 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 9ba2734cd39f:46195 this server is in the failed servers list 2024-12-04T19:24:52,958 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,46195,1733340273483, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,46195,1733340273483, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=9ba2734cd39f:46195 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9ba2734cd39f:46195 2024-12-04T19:24:52,958 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,46195,1733340273483, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 9ba2734cd39f:46195 2024-12-04T19:24:52,958 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,46195,1733340273483, seqNum=18 from cache 2024-12-04T19:24:52,961 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:52,961 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 88a3d8cc28447f8692a6ef68159edd5f, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:24:52,961 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:52,961 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:52,961 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:52,961 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:52,963 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:52,964 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88a3d8cc28447f8692a6ef68159edd5f columnFamilyName cf1 2024-12-04T19:24:52,964 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:52,969 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf1/e0a1a2e721fe4459b572121da574a18c 2024-12-04T19:24:52,969 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(327): Store=88a3d8cc28447f8692a6ef68159edd5f/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:52,970 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:52,970 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88a3d8cc28447f8692a6ef68159edd5f columnFamilyName cf2 2024-12-04T19:24:52,970 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:52,977 DEBUG [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/cf2/9032ae5e5a524ac9ab93d3e53caf3798 2024-12-04T19:24:52,977 INFO [StoreOpener-88a3d8cc28447f8692a6ef68159edd5f-1 {}] regionserver.HStore(327): Store=88a3d8cc28447f8692a6ef68159edd5f/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:52,977 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:52,978 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:52,979 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:52,980 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-04T19:24:52,980 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-04T19:24:52,980 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:52,980 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:52,981 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T19:24:52,981 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-04T19:24:52,981 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-04T19:24:52,982 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:52,984 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 88a3d8cc28447f8692a6ef68159edd5f; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63024484, jitterRate=-0.06086200475692749}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T19:24:52,984 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:24:52,984 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 88a3d8cc28447f8692a6ef68159edd5f: Running coprocessor pre-open hook at 1733340292961Writing region info on filesystem at 1733340292961Initializing all the Stores at 1733340292962 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340292962Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340292963 (+1 ms)Cleaning up temporary data from old regions at 1733340292980 (+17 ms)Running coprocessor post-open hooks at 1733340292984 (+4 ms)Region opened successfully at 1733340292984 2024-12-04T19:24:52,985 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., pid=21, masterSystemTime=1733340292957 2024-12-04T19:24:52,988 DEBUG [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:52,988 INFO [RS_OPEN_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:24:52,988 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=88a3d8cc28447f8692a6ef68159edd5f, regionState=OPEN, openSeqNum=18, regionLocation=9ba2734cd39f,42371,1733340273381 2024-12-04T19:24:52,990 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88a3d8cc28447f8692a6ef68159edd5f, server=9ba2734cd39f,42371,1733340273381 because future has completed 2024-12-04T19:24:52,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-04T19:24:52,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 88a3d8cc28447f8692a6ef68159edd5f, server=9ba2734cd39f,42371,1733340273381 in 188 msec 2024-12-04T19:24:52,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-12-04T19:24:52,996 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 9ba2734cd39f,46195,1733340273483 after splitting done 2024-12-04T19:24:52,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=88a3d8cc28447f8692a6ef68159edd5f, ASSIGN in 461 msec 2024-12-04T19:24:52,998 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 9ba2734cd39f,46195,1733340273483, splitWal=true, meta=true in 1.5770 sec 2024-12-04T19:24:53,240 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T19:24:53,474 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., hostname=9ba2734cd39f,42371,1733340273381, seqNum=18] 2024-12-04T19:24:53,490 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=408 (was 414), OpenFileDescriptor=1046 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=166 (was 180), ProcessCount=11 (was 11), AvailableMemoryMB=15585 (was 15652) 2024-12-04T19:24:53,493 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1046 is superior to 1024 2024-12-04T19:24:53,506 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=408, OpenFileDescriptor=1046, MaxFileDescriptor=1048576, SystemLoadAverage=166, ProcessCount=11, AvailableMemoryMB=15584 2024-12-04T19:24:53,506 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1046 is superior to 1024 2024-12-04T19:24:53,522 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:53,523 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:53,524 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:24:53,526 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-26727282, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/hregion-26727282, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:24:53,537 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-26727282/hregion-26727282.1733340293526, exclude list is [], retry=0 2024-12-04T19:24:53,539 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:53,539 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:53,539 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:53,561 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-26727282/hregion-26727282.1733340293526 2024-12-04T19:24:53,562 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:37675:37675)] 2024-12-04T19:24:53,562 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b55966e8ca904d8449a51de2b6b42c54, NAME => 'testReplayEditsWrittenViaHRegion,,1733340293522.b55966e8ca904d8449a51de2b6b42c54.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33439/hbase 2024-12-04T19:24:53,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741898_1076 (size=67) 2024-12-04T19:24:53,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741898_1076 (size=67) 2024-12-04T19:24:53,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741898_1076 (size=67) 2024-12-04T19:24:53,572 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733340293522.b55966e8ca904d8449a51de2b6b42c54.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:53,573 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,574 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b55966e8ca904d8449a51de2b6b42c54 columnFamilyName a 2024-12-04T19:24:53,574 DEBUG [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:53,574 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] regionserver.HStore(327): Store=b55966e8ca904d8449a51de2b6b42c54/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:53,574 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,576 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b55966e8ca904d8449a51de2b6b42c54 columnFamilyName b 2024-12-04T19:24:53,576 DEBUG [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:53,576 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] regionserver.HStore(327): Store=b55966e8ca904d8449a51de2b6b42c54/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:53,576 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,577 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b55966e8ca904d8449a51de2b6b42c54 columnFamilyName c 2024-12-04T19:24:53,577 DEBUG [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:53,578 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] regionserver.HStore(327): Store=b55966e8ca904d8449a51de2b6b42c54/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:53,578 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,579 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,579 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,580 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,580 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,580 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:24:53,581 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,583 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:24:53,584 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b55966e8ca904d8449a51de2b6b42c54; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60191533, jitterRate=-0.1030762642621994}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:24:53,584 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b55966e8ca904d8449a51de2b6b42c54: Writing region info on filesystem at 1733340293572Initializing all the Stores at 1733340293572Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340293572Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340293573 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340293573Cleaning up temporary data from old regions at 1733340293580 (+7 ms)Region opened successfully at 1733340293584 (+4 ms) 2024-12-04T19:24:53,585 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b55966e8ca904d8449a51de2b6b42c54, disabling compactions & flushes 2024-12-04T19:24:53,585 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733340293522.b55966e8ca904d8449a51de2b6b42c54. 2024-12-04T19:24:53,585 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733340293522.b55966e8ca904d8449a51de2b6b42c54. 2024-12-04T19:24:53,585 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733340293522.b55966e8ca904d8449a51de2b6b42c54. after waiting 0 ms 2024-12-04T19:24:53,585 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733340293522.b55966e8ca904d8449a51de2b6b42c54. 2024-12-04T19:24:53,585 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733340293522.b55966e8ca904d8449a51de2b6b42c54. 2024-12-04T19:24:53,585 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b55966e8ca904d8449a51de2b6b42c54: Waiting for close lock at 1733340293584Disabling compacts and flushes for region at 1733340293584Disabling writes for close at 1733340293585 (+1 ms)Writing region close event to WAL at 1733340293585Closed at 1733340293585 2024-12-04T19:24:53,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741897_1075 (size=175) 2024-12-04T19:24:53,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741897_1075 (size=175) 2024-12-04T19:24:53,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741897_1075 (size=175) 2024-12-04T19:24:53,590 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:24:53,590 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-26727282:(num 1733340293526) 2024-12-04T19:24:53,590 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:24:53,592 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:24:53,603 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521/wal.1733340293592, exclude list is [], retry=0 2024-12-04T19:24:53,605 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:53,606 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:53,606 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:53,628 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521/wal.1733340293592 2024-12-04T19:24:53,628 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:42335:42335)] 2024-12-04T19:24:53,628 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b55966e8ca904d8449a51de2b6b42c54, NAME => 'testReplayEditsWrittenViaHRegion,,1733340293522.b55966e8ca904d8449a51de2b6b42c54.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:24:53,628 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733340293522.b55966e8ca904d8449a51de2b6b42c54.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:53,628 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,628 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,630 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,630 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b55966e8ca904d8449a51de2b6b42c54 columnFamilyName a 2024-12-04T19:24:53,630 DEBUG [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:53,631 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] regionserver.HStore(327): Store=b55966e8ca904d8449a51de2b6b42c54/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:53,631 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,631 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b55966e8ca904d8449a51de2b6b42c54 columnFamilyName b 2024-12-04T19:24:53,631 DEBUG [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:53,632 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] regionserver.HStore(327): Store=b55966e8ca904d8449a51de2b6b42c54/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:53,632 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,632 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b55966e8ca904d8449a51de2b6b42c54 columnFamilyName c 2024-12-04T19:24:53,632 DEBUG [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:53,633 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] regionserver.HStore(327): Store=b55966e8ca904d8449a51de2b6b42c54/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:53,633 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,634 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,635 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,635 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,635 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,636 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:24:53,637 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,637 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b55966e8ca904d8449a51de2b6b42c54; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60806694, jitterRate=-0.09390965104103088}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:24:53,638 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b55966e8ca904d8449a51de2b6b42c54: Writing region info on filesystem at 1733340293628Initializing all the Stores at 1733340293629 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340293629Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340293629Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340293629Cleaning up temporary data from old regions at 1733340293635 (+6 ms)Region opened successfully at 1733340293638 (+3 ms) 2024-12-04T19:24:53,665 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b55966e8ca904d8449a51de2b6b42c54 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-04T19:24:53,680 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/.tmp/a/59168517adc04a8a99b4f12e2d970f0a is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733340293638/Put/seqid=0 2024-12-04T19:24:53,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741900_1078 (size=5958) 2024-12-04T19:24:53,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741900_1078 (size=5958) 2024-12-04T19:24:53,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741900_1078 (size=5958) 2024-12-04T19:24:53,687 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/.tmp/a/59168517adc04a8a99b4f12e2d970f0a 2024-12-04T19:24:53,712 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/.tmp/b/0de9b35eaa9a45dfae2b5131b363ff45 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733340293648/Put/seqid=0 2024-12-04T19:24:53,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741901_1079 (size=5958) 2024-12-04T19:24:53,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741901_1079 (size=5958) 2024-12-04T19:24:53,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741901_1079 (size=5958) 2024-12-04T19:24:53,718 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/.tmp/b/0de9b35eaa9a45dfae2b5131b363ff45 2024-12-04T19:24:53,739 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/.tmp/c/e6c36096a3934cc3959eceea5d95dae4 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733340293656/Put/seqid=0 2024-12-04T19:24:53,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741902_1080 (size=5958) 2024-12-04T19:24:53,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741902_1080 (size=5958) 2024-12-04T19:24:53,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741902_1080 (size=5958) 2024-12-04T19:24:53,746 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/.tmp/c/e6c36096a3934cc3959eceea5d95dae4 2024-12-04T19:24:53,752 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/.tmp/a/59168517adc04a8a99b4f12e2d970f0a as hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/a/59168517adc04a8a99b4f12e2d970f0a 2024-12-04T19:24:53,757 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/a/59168517adc04a8a99b4f12e2d970f0a, entries=10, sequenceid=33, filesize=5.8 K 2024-12-04T19:24:53,758 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/.tmp/b/0de9b35eaa9a45dfae2b5131b363ff45 as hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/b/0de9b35eaa9a45dfae2b5131b363ff45 2024-12-04T19:24:53,763 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/b/0de9b35eaa9a45dfae2b5131b363ff45, entries=10, sequenceid=33, filesize=5.8 K 2024-12-04T19:24:53,764 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/.tmp/c/e6c36096a3934cc3959eceea5d95dae4 as hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/c/e6c36096a3934cc3959eceea5d95dae4 2024-12-04T19:24:53,769 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/c/e6c36096a3934cc3959eceea5d95dae4, entries=10, sequenceid=33, filesize=5.8 K 2024-12-04T19:24:53,770 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for b55966e8ca904d8449a51de2b6b42c54 in 105ms, sequenceid=33, compaction requested=false 2024-12-04T19:24:53,770 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b55966e8ca904d8449a51de2b6b42c54: 2024-12-04T19:24:53,770 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b55966e8ca904d8449a51de2b6b42c54, disabling compactions & flushes 2024-12-04T19:24:53,770 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733340293522.b55966e8ca904d8449a51de2b6b42c54. 2024-12-04T19:24:53,770 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733340293522.b55966e8ca904d8449a51de2b6b42c54. 2024-12-04T19:24:53,770 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733340293522.b55966e8ca904d8449a51de2b6b42c54. after waiting 0 ms 2024-12-04T19:24:53,770 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733340293522.b55966e8ca904d8449a51de2b6b42c54. 2024-12-04T19:24:53,772 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733340293522.b55966e8ca904d8449a51de2b6b42c54. 2024-12-04T19:24:53,772 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b55966e8ca904d8449a51de2b6b42c54: Waiting for close lock at 1733340293770Disabling compacts and flushes for region at 1733340293770Disabling writes for close at 1733340293770Writing region close event to WAL at 1733340293771 (+1 ms)Closed at 1733340293771 2024-12-04T19:24:53,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741899_1077 (size=6311) 2024-12-04T19:24:53,774 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521/wal.1733340293592 not finished, retry = 0 2024-12-04T19:24:53,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741899_1077 (size=6311) 2024-12-04T19:24:53,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741899_1077 (size=6311) 2024-12-04T19:24:53,878 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/b/0de9b35eaa9a45dfae2b5131b363ff45 to hdfs://localhost:33439/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/b/0de9b35eaa9a45dfae2b5131b363ff45 2024-12-04T19:24:53,891 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33439/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521/wal.1733340293592, size=6.2 K (6311bytes) 2024-12-04T19:24:53,892 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33439/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521/wal.1733340293592 2024-12-04T19:24:53,892 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33439/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521/wal.1733340293592 after 0ms 2024-12-04T19:24:53,915 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:53,915 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33439/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521/wal.1733340293592 took 24ms 2024-12-04T19:24:53,917 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33439/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521/wal.1733340293592 so closing down 2024-12-04T19:24:53,917 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:24:53,918 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733340293592.temp 2024-12-04T19:24:53,939 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/recovered.edits/0000000000000000003-wal.1733340293592.temp 2024-12-04T19:24:53,939 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:24:53,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741903_1081 (size=5575) 2024-12-04T19:24:53,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741903_1081 (size=5575) 2024-12-04T19:24:53,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741903_1081 (size=5575) 2024-12-04T19:24:53,946 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/recovered.edits/0000000000000000003-wal.1733340293592.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-04T19:24:53,947 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/recovered.edits/0000000000000000003-wal.1733340293592.temp to hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/recovered.edits/0000000000000000032 2024-12-04T19:24:53,947 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 32 ms; skipped=2; WAL=hdfs://localhost:33439/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521/wal.1733340293592, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-12-04T19:24:53,947 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33439/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521/wal.1733340293592, journal: Splitting hdfs://localhost:33439/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521/wal.1733340293592, size=6.2 K (6311bytes) at 1733340293892Finishing writing output for hdfs://localhost:33439/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521/wal.1733340293592 so closing down at 1733340293917 (+25 ms)Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/recovered.edits/0000000000000000003-wal.1733340293592.temp at 1733340293939 (+22 ms)3 split writer threads finished at 1733340293939Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/recovered.edits/0000000000000000003-wal.1733340293592.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733340293946 (+7 ms)Rename recovered edits hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/recovered.edits/0000000000000000003-wal.1733340293592.temp to hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/recovered.edits/0000000000000000032 at 1733340293947 (+1 ms)Processed 32 edits across 1 Regions in 32 ms; skipped=2; WAL=hdfs://localhost:33439/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521/wal.1733340293592, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1733340293947 2024-12-04T19:24:53,949 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33439/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521/wal.1733340293592 to hdfs://localhost:33439/hbase/oldWALs/wal.1733340293592 2024-12-04T19:24:53,950 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/recovered.edits/0000000000000000032 2024-12-04T19:24:53,950 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:24:53,952 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:24:53,962 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521/wal.1733340293952, exclude list is [], retry=0 2024-12-04T19:24:53,965 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:53,965 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:53,965 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:53,988 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521/wal.1733340293952 2024-12-04T19:24:53,988 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:42335:42335)] 2024-12-04T19:24:53,988 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b55966e8ca904d8449a51de2b6b42c54, NAME => 'testReplayEditsWrittenViaHRegion,,1733340293522.b55966e8ca904d8449a51de2b6b42c54.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:24:53,988 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733340293522.b55966e8ca904d8449a51de2b6b42c54.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:53,988 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,988 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,990 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,991 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b55966e8ca904d8449a51de2b6b42c54 columnFamilyName a 2024-12-04T19:24:53,991 DEBUG [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:53,996 DEBUG [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/a/59168517adc04a8a99b4f12e2d970f0a 2024-12-04T19:24:53,997 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] regionserver.HStore(327): Store=b55966e8ca904d8449a51de2b6b42c54/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:53,997 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:53,998 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b55966e8ca904d8449a51de2b6b42c54 columnFamilyName b 2024-12-04T19:24:53,998 DEBUG [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:53,999 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] regionserver.HStore(327): Store=b55966e8ca904d8449a51de2b6b42c54/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:53,999 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:54,000 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b55966e8ca904d8449a51de2b6b42c54 columnFamilyName c 2024-12-04T19:24:54,000 DEBUG [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:54,005 DEBUG [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/c/e6c36096a3934cc3959eceea5d95dae4 2024-12-04T19:24:54,005 INFO [StoreOpener-b55966e8ca904d8449a51de2b6b42c54-1 {}] regionserver.HStore(327): Store=b55966e8ca904d8449a51de2b6b42c54/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:54,005 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:54,006 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:54,007 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:54,008 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/recovered.edits/0000000000000000032 2024-12-04T19:24:54,031 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:24:54,032 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/recovered.edits/0000000000000000032 2024-12-04T19:24:54,033 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b55966e8ca904d8449a51de2b6b42c54 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-04T19:24:54,045 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/.tmp/b/f1e8296518604481b630ddfd004263f9 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733340293648/Put/seqid=0 2024-12-04T19:24:54,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741905_1083 (size=5958) 2024-12-04T19:24:54,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741905_1083 (size=5958) 2024-12-04T19:24:54,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741905_1083 (size=5958) 2024-12-04T19:24:54,055 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/.tmp/b/f1e8296518604481b630ddfd004263f9 2024-12-04T19:24:54,061 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/.tmp/b/f1e8296518604481b630ddfd004263f9 as hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/b/f1e8296518604481b630ddfd004263f9 2024-12-04T19:24:54,066 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/b/f1e8296518604481b630ddfd004263f9, entries=10, sequenceid=32, filesize=5.8 K 2024-12-04T19:24:54,067 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for b55966e8ca904d8449a51de2b6b42c54 in 34ms, sequenceid=32, compaction requested=false; wal=null 2024-12-04T19:24:54,067 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/recovered.edits/0000000000000000032 2024-12-04T19:24:54,068 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:54,068 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:54,069 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:24:54,070 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b55966e8ca904d8449a51de2b6b42c54 2024-12-04T19:24:54,073 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/b55966e8ca904d8449a51de2b6b42c54/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-04T19:24:54,074 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b55966e8ca904d8449a51de2b6b42c54; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61348595, jitterRate=-0.08583469688892365}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:24:54,074 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b55966e8ca904d8449a51de2b6b42c54: Writing region info on filesystem at 1733340293989Initializing all the Stores at 1733340293990 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340293990Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340293990Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340293990Obtaining lock to block concurrent updates at 1733340294033 (+43 ms)Preparing flush snapshotting stores in b55966e8ca904d8449a51de2b6b42c54 at 1733340294033Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733340293522.b55966e8ca904d8449a51de2b6b42c54., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733340294033Flushing stores of testReplayEditsWrittenViaHRegion,,1733340293522.b55966e8ca904d8449a51de2b6b42c54. at 1733340294033Flushing b55966e8ca904d8449a51de2b6b42c54/b: creating writer at 1733340294033Flushing b55966e8ca904d8449a51de2b6b42c54/b: appending metadata at 1733340294045 (+12 ms)Flushing b55966e8ca904d8449a51de2b6b42c54/b: closing flushed file at 1733340294045Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c0de37e: reopening flushed file at 1733340294060 (+15 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for b55966e8ca904d8449a51de2b6b42c54 in 34ms, sequenceid=32, compaction requested=false; wal=null at 1733340294067 (+7 ms)Cleaning up temporary data from old regions at 1733340294068 (+1 ms)Region opened successfully at 1733340294074 (+6 ms) 2024-12-04T19:24:54,094 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=417 (was 408) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:33450 [Waiting for operation #56] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1762372386-172.17.0.2-1733340268509:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:44608 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:54336 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:45420 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:51304 [Waiting for operation #34] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1762372386-172.17.0.2-1733340268509:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:45494 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1762372386-172.17.0.2-1733340268509:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1122 (was 1046) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=166 (was 166), ProcessCount=11 (was 11), AvailableMemoryMB=15577 (was 15584) 2024-12-04T19:24:54,094 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1122 is superior to 1024 2024-12-04T19:24:54,105 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=417, OpenFileDescriptor=1122, MaxFileDescriptor=1048576, SystemLoadAverage=166, ProcessCount=11, AvailableMemoryMB=15576 2024-12-04T19:24:54,105 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1122 is superior to 1024 2024-12-04T19:24:54,118 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:54,119 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:24:54,120 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:24:54,122 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-71230317, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/hregion-71230317, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:24:54,133 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-71230317/hregion-71230317.1733340294122, exclude list is [], retry=0 2024-12-04T19:24:54,135 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:54,135 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:54,136 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:54,161 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-71230317/hregion-71230317.1733340294122 2024-12-04T19:24:54,162 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:33421:33421)] 2024-12-04T19:24:54,162 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d82fda38e113d75164ffc5986ab9cbb7, NAME => 'testReplayEditsAfterAbortingFlush,,1733340294118.d82fda38e113d75164ffc5986ab9cbb7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33439/hbase 2024-12-04T19:24:54,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741907_1085 (size=68) 2024-12-04T19:24:54,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741907_1085 (size=68) 2024-12-04T19:24:54,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741907_1085 (size=68) 2024-12-04T19:24:54,573 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733340294118.d82fda38e113d75164ffc5986ab9cbb7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:54,577 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,580 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d82fda38e113d75164ffc5986ab9cbb7 columnFamilyName a 2024-12-04T19:24:54,580 DEBUG [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:54,581 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] regionserver.HStore(327): Store=d82fda38e113d75164ffc5986ab9cbb7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:54,581 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,583 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d82fda38e113d75164ffc5986ab9cbb7 columnFamilyName b 2024-12-04T19:24:54,583 DEBUG [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:54,584 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] regionserver.HStore(327): Store=d82fda38e113d75164ffc5986ab9cbb7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:54,584 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,585 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d82fda38e113d75164ffc5986ab9cbb7 columnFamilyName c 2024-12-04T19:24:54,585 DEBUG [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:54,586 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] regionserver.HStore(327): Store=d82fda38e113d75164ffc5986ab9cbb7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:54,586 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,587 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,587 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,588 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,588 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,589 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:24:54,590 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,592 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:24:54,593 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d82fda38e113d75164ffc5986ab9cbb7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69268977, jitterRate=0.03218819200992584}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:24:54,594 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d82fda38e113d75164ffc5986ab9cbb7: Writing region info on filesystem at 1733340294573Initializing all the Stores at 1733340294576 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340294576Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340294577 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340294577Cleaning up temporary data from old regions at 1733340294588 (+11 ms)Region opened successfully at 1733340294593 (+5 ms) 2024-12-04T19:24:54,594 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d82fda38e113d75164ffc5986ab9cbb7, disabling compactions & flushes 2024-12-04T19:24:54,594 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733340294118.d82fda38e113d75164ffc5986ab9cbb7. 2024-12-04T19:24:54,594 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733340294118.d82fda38e113d75164ffc5986ab9cbb7. 2024-12-04T19:24:54,594 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733340294118.d82fda38e113d75164ffc5986ab9cbb7. after waiting 0 ms 2024-12-04T19:24:54,594 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733340294118.d82fda38e113d75164ffc5986ab9cbb7. 2024-12-04T19:24:54,594 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733340294118.d82fda38e113d75164ffc5986ab9cbb7. 2024-12-04T19:24:54,594 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d82fda38e113d75164ffc5986ab9cbb7: Waiting for close lock at 1733340294594Disabling compacts and flushes for region at 1733340294594Disabling writes for close at 1733340294594Writing region close event to WAL at 1733340294594Closed at 1733340294594 2024-12-04T19:24:54,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741906_1084 (size=175) 2024-12-04T19:24:54,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741906_1084 (size=175) 2024-12-04T19:24:54,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741906_1084 (size=175) 2024-12-04T19:24:54,599 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:24:54,599 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-71230317:(num 1733340294122) 2024-12-04T19:24:54,599 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:24:54,600 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340294117, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:24:54,611 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340294117/wal.1733340294601, exclude list is [], retry=0 2024-12-04T19:24:54,614 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:24:54,614 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:24:54,614 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:24:54,637 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340294117/wal.1733340294601 2024-12-04T19:24:54,638 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:42335:42335)] 2024-12-04T19:24:54,690 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d82fda38e113d75164ffc5986ab9cbb7, NAME => 'testReplayEditsAfterAbortingFlush,,1733340294118.d82fda38e113d75164ffc5986ab9cbb7.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:24:54,692 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,692 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733340294118.d82fda38e113d75164ffc5986ab9cbb7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:24:54,692 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,692 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,694 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,695 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d82fda38e113d75164ffc5986ab9cbb7 columnFamilyName a 2024-12-04T19:24:54,696 DEBUG [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:54,696 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] regionserver.HStore(327): Store=d82fda38e113d75164ffc5986ab9cbb7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:54,696 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,697 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d82fda38e113d75164ffc5986ab9cbb7 columnFamilyName b 2024-12-04T19:24:54,697 DEBUG [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:54,698 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] regionserver.HStore(327): Store=d82fda38e113d75164ffc5986ab9cbb7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:54,698 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,699 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d82fda38e113d75164ffc5986ab9cbb7 columnFamilyName c 2024-12-04T19:24:54,699 DEBUG [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:24:54,699 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] regionserver.HStore(327): Store=d82fda38e113d75164ffc5986ab9cbb7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:24:54,699 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,700 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,701 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,702 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,702 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,703 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:24:54,704 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,705 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d82fda38e113d75164ffc5986ab9cbb7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62955770, jitterRate=-0.06188592314720154}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:24:54,705 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:24:54,706 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d82fda38e113d75164ffc5986ab9cbb7: Running coprocessor pre-open hook at 1733340294693Writing region info on filesystem at 1733340294693Initializing all the Stores at 1733340294694 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340294694Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340294694Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340294694Cleaning up temporary data from old regions at 1733340294702 (+8 ms)Running coprocessor post-open hooks at 1733340294705 (+3 ms)Region opened successfully at 1733340294706 (+1 ms) 2024-12-04T19:24:54,719 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d82fda38e113d75164ffc5986ab9cbb7 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-04T19:24:54,720 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d82fda38e113d75164ffc5986ab9cbb7/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:24:54,846 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T19:24:55,721 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d82fda38e113d75164ffc5986ab9cbb7/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:24:56,722 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d82fda38e113d75164ffc5986ab9cbb7/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:24:57,724 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d82fda38e113d75164ffc5986ab9cbb7/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:24:58,725 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d82fda38e113d75164ffc5986ab9cbb7/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:24:59,726 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d82fda38e113d75164ffc5986ab9cbb7/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:25:00,195 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T19:25:00,727 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d82fda38e113d75164ffc5986ab9cbb7/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:25:01,728 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d82fda38e113d75164ffc5986ab9cbb7/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:25:01,844 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T19:25:02,729 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d82fda38e113d75164ffc5986ab9cbb7/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:25:02,980 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-04T19:25:02,980 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-04T19:25:02,981 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-04T19:25:02,981 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-04T19:25:02,982 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-04T19:25:02,982 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-04T19:25:03,730 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d82fda38e113d75164ffc5986ab9cbb7/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:25:03,732 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d82fda38e113d75164ffc5986ab9cbb7: 2024-12-04T19:25:03,732 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733340294118.d82fda38e113d75164ffc5986ab9cbb7. 2024-12-04T19:25:03,746 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d82fda38e113d75164ffc5986ab9cbb7: 2024-12-04T19:25:03,747 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-04T19:25:03,747 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d82fda38e113d75164ffc5986ab9cbb7, disabling compactions & flushes 2024-12-04T19:25:03,747 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733340294118.d82fda38e113d75164ffc5986ab9cbb7. 2024-12-04T19:25:03,747 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733340294118.d82fda38e113d75164ffc5986ab9cbb7. 2024-12-04T19:25:03,747 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733340294118.d82fda38e113d75164ffc5986ab9cbb7. after waiting 0 ms 2024-12-04T19:25:03,747 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733340294118.d82fda38e113d75164ffc5986ab9cbb7. 2024-12-04T19:25:03,747 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733340294118.d82fda38e113d75164ffc5986ab9cbb7. 2024-12-04T19:25:03,747 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733340294118.d82fda38e113d75164ffc5986ab9cbb7. 2024-12-04T19:25:03,747 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d82fda38e113d75164ffc5986ab9cbb7: Waiting for close lock at 1733340303747Running coprocessor pre-close hooks at 1733340303747Disabling compacts and flushes for region at 1733340303747Disabling writes for close at 1733340303747Writing region close event to WAL at 1733340303747Running coprocessor post-close hooks at 1733340303747Closed at 1733340303747 2024-12-04T19:25:03,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741908_1086 (size=4236) 2024-12-04T19:25:03,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741908_1086 (size=4236) 2024-12-04T19:25:03,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741908_1086 (size=4236) 2024-12-04T19:25:03,762 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33439/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340294117/wal.1733340294601, size=4.1 K (4236bytes) 2024-12-04T19:25:03,762 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33439/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340294117/wal.1733340294601 2024-12-04T19:25:03,763 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33439/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340294117/wal.1733340294601 after 1ms 2024-12-04T19:25:03,785 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:25:03,785 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33439/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340294117/wal.1733340294601 took 23ms 2024-12-04T19:25:03,787 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33439/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340294117/wal.1733340294601 so closing down 2024-12-04T19:25:03,787 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:25:03,788 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733340294601.temp 2024-12-04T19:25:03,810 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/recovered.edits/0000000000000000004-wal.1733340294601.temp 2024-12-04T19:25:03,810 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:25:03,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741909_1087 (size=3245) 2024-12-04T19:25:03,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741909_1087 (size=3245) 2024-12-04T19:25:03,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741909_1087 (size=3245) 2024-12-04T19:25:03,846 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/recovered.edits/0000000000000000004-wal.1733340294601.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-04T19:25:03,848 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/recovered.edits/0000000000000000004-wal.1733340294601.temp to hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/recovered.edits/0000000000000000026 2024-12-04T19:25:03,848 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 63 ms; skipped=3; WAL=hdfs://localhost:33439/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340294117/wal.1733340294601, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-12-04T19:25:03,848 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33439/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340294117/wal.1733340294601, journal: Splitting hdfs://localhost:33439/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340294117/wal.1733340294601, size=4.1 K (4236bytes) at 1733340303762Finishing writing output for hdfs://localhost:33439/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340294117/wal.1733340294601 so closing down at 1733340303787 (+25 ms)Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/recovered.edits/0000000000000000004-wal.1733340294601.temp at 1733340303810 (+23 ms)3 split writer threads finished at 1733340303811 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/recovered.edits/0000000000000000004-wal.1733340294601.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733340303846 (+35 ms)Rename recovered edits hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/recovered.edits/0000000000000000004-wal.1733340294601.temp to hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/recovered.edits/0000000000000000026 at 1733340303848 (+2 ms)Processed 23 edits across 1 Regions in 63 ms; skipped=3; WAL=hdfs://localhost:33439/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340294117/wal.1733340294601, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1733340303848 2024-12-04T19:25:03,850 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33439/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340294117/wal.1733340294601 to hdfs://localhost:33439/hbase/oldWALs/wal.1733340294601 2024-12-04T19:25:03,851 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/recovered.edits/0000000000000000026 2024-12-04T19:25:03,851 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:25:03,853 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340294117, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:25:03,869 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340294117/wal.1733340303854, exclude list is [], retry=0 2024-12-04T19:25:03,871 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:25:03,871 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:25:03,872 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:25:03,892 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340294117/wal.1733340303854 2024-12-04T19:25:03,893 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:33421:33421)] 2024-12-04T19:25:03,893 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d82fda38e113d75164ffc5986ab9cbb7, NAME => 'testReplayEditsAfterAbortingFlush,,1733340294118.d82fda38e113d75164ffc5986ab9cbb7.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:25:03,893 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:25:03,894 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733340294118.d82fda38e113d75164ffc5986ab9cbb7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:25:03,894 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:25:03,894 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:25:03,895 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:25:03,896 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d82fda38e113d75164ffc5986ab9cbb7 columnFamilyName a 2024-12-04T19:25:03,896 DEBUG [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:03,897 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] regionserver.HStore(327): Store=d82fda38e113d75164ffc5986ab9cbb7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:03,897 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:25:03,897 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d82fda38e113d75164ffc5986ab9cbb7 columnFamilyName b 2024-12-04T19:25:03,897 DEBUG [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:03,898 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] regionserver.HStore(327): Store=d82fda38e113d75164ffc5986ab9cbb7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:03,898 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:25:03,899 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d82fda38e113d75164ffc5986ab9cbb7 columnFamilyName c 2024-12-04T19:25:03,899 DEBUG [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:03,899 INFO [StoreOpener-d82fda38e113d75164ffc5986ab9cbb7-1 {}] regionserver.HStore(327): Store=d82fda38e113d75164ffc5986ab9cbb7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:03,899 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:25:03,900 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:25:03,901 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:25:03,902 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/recovered.edits/0000000000000000026 2024-12-04T19:25:03,946 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:25:03,947 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/recovered.edits/0000000000000000026 2024-12-04T19:25:03,948 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d82fda38e113d75164ffc5986ab9cbb7 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-04T19:25:03,960 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/.tmp/a/0eaf002933554a1eb827e5ad001e6bbc is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733340303738/Put/seqid=0 2024-12-04T19:25:03,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741911_1089 (size=5523) 2024-12-04T19:25:03,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741911_1089 (size=5523) 2024-12-04T19:25:03,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741911_1089 (size=5523) 2024-12-04T19:25:03,967 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/.tmp/a/0eaf002933554a1eb827e5ad001e6bbc 2024-12-04T19:25:03,984 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/.tmp/b/3af10940e36140179879b32ed093cdc0 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733340303733/Put/seqid=0 2024-12-04T19:25:03,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741912_1090 (size=5524) 2024-12-04T19:25:03,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741912_1090 (size=5524) 2024-12-04T19:25:03,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741912_1090 (size=5524) 2024-12-04T19:25:03,990 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/.tmp/b/3af10940e36140179879b32ed093cdc0 2024-12-04T19:25:04,008 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/.tmp/c/7666a7905f7448bd959308280b7ab97a is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733340303735/Put/seqid=0 2024-12-04T19:25:04,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741913_1091 (size=5457) 2024-12-04T19:25:04,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741913_1091 (size=5457) 2024-12-04T19:25:04,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741913_1091 (size=5457) 2024-12-04T19:25:04,015 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/.tmp/c/7666a7905f7448bd959308280b7ab97a 2024-12-04T19:25:04,020 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/.tmp/a/0eaf002933554a1eb827e5ad001e6bbc as hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/a/0eaf002933554a1eb827e5ad001e6bbc 2024-12-04T19:25:04,026 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/a/0eaf002933554a1eb827e5ad001e6bbc, entries=7, sequenceid=26, filesize=5.4 K 2024-12-04T19:25:04,028 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/.tmp/b/3af10940e36140179879b32ed093cdc0 as hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/b/3af10940e36140179879b32ed093cdc0 2024-12-04T19:25:04,033 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/b/3af10940e36140179879b32ed093cdc0, entries=7, sequenceid=26, filesize=5.4 K 2024-12-04T19:25:04,034 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/.tmp/c/7666a7905f7448bd959308280b7ab97a as hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/c/7666a7905f7448bd959308280b7ab97a 2024-12-04T19:25:04,040 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/c/7666a7905f7448bd959308280b7ab97a, entries=6, sequenceid=26, filesize=5.3 K 2024-12-04T19:25:04,040 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for d82fda38e113d75164ffc5986ab9cbb7 in 92ms, sequenceid=26, compaction requested=false; wal=null 2024-12-04T19:25:04,041 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/recovered.edits/0000000000000000026 2024-12-04T19:25:04,043 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:25:04,043 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:25:04,043 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:25:04,045 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:25:04,047 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testReplayEditsAfterAbortingFlush/d82fda38e113d75164ffc5986ab9cbb7/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-04T19:25:04,048 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d82fda38e113d75164ffc5986ab9cbb7; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69492299, jitterRate=0.03551594913005829}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:25:04,048 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d82fda38e113d75164ffc5986ab9cbb7 2024-12-04T19:25:04,048 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d82fda38e113d75164ffc5986ab9cbb7: Running coprocessor pre-open hook at 1733340303894Writing region info on filesystem at 1733340303894Initializing all the Stores at 1733340303895 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340303895Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340303895Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340303895Obtaining lock to block concurrent updates at 1733340303948 (+53 ms)Preparing flush snapshotting stores in d82fda38e113d75164ffc5986ab9cbb7 at 1733340303948Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733340294118.d82fda38e113d75164ffc5986ab9cbb7., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733340303948Flushing stores of testReplayEditsAfterAbortingFlush,,1733340294118.d82fda38e113d75164ffc5986ab9cbb7. at 1733340303948Flushing d82fda38e113d75164ffc5986ab9cbb7/a: creating writer at 1733340303948Flushing d82fda38e113d75164ffc5986ab9cbb7/a: appending metadata at 1733340303960 (+12 ms)Flushing d82fda38e113d75164ffc5986ab9cbb7/a: closing flushed file at 1733340303960Flushing d82fda38e113d75164ffc5986ab9cbb7/b: creating writer at 1733340303971 (+11 ms)Flushing d82fda38e113d75164ffc5986ab9cbb7/b: appending metadata at 1733340303983 (+12 ms)Flushing d82fda38e113d75164ffc5986ab9cbb7/b: closing flushed file at 1733340303983Flushing d82fda38e113d75164ffc5986ab9cbb7/c: creating writer at 1733340303995 (+12 ms)Flushing d82fda38e113d75164ffc5986ab9cbb7/c: appending metadata at 1733340304007 (+12 ms)Flushing d82fda38e113d75164ffc5986ab9cbb7/c: closing flushed file at 1733340304008 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bd1623b: reopening flushed file at 1733340304020 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35b3a642: reopening flushed file at 1733340304027 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65f99299: reopening flushed file at 1733340304033 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for d82fda38e113d75164ffc5986ab9cbb7 in 92ms, sequenceid=26, compaction requested=false; wal=null at 1733340304040 (+7 ms)Cleaning up temporary data from old regions at 1733340304043 (+3 ms)Running coprocessor post-open hooks at 1733340304048 (+5 ms)Region opened successfully at 1733340304048 2024-12-04T19:25:04,071 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415 (was 417), OpenFileDescriptor=1180 (was 1122) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=148 (was 166), ProcessCount=11 (was 11), AvailableMemoryMB=15537 (was 15576) 2024-12-04T19:25:04,071 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-12-04T19:25:04,082 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=415, OpenFileDescriptor=1180, MaxFileDescriptor=1048576, SystemLoadAverage=148, ProcessCount=11, AvailableMemoryMB=15536 2024-12-04T19:25:04,082 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-12-04T19:25:04,095 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:25:04,097 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:25:04,097 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:25:04,100 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-79153010, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/hregion-79153010, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:25:04,112 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-79153010/hregion-79153010.1733340304100, exclude list is [], retry=0 2024-12-04T19:25:04,115 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:25:04,115 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:25:04,115 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:25:04,141 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-79153010/hregion-79153010.1733340304100 2024-12-04T19:25:04,141 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:37675:37675)] 2024-12-04T19:25:04,141 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 042e7b11f21e089c66c76494e91abf49, NAME => 'testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33439/hbase 2024-12-04T19:25:04,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741915_1093 (size=61) 2024-12-04T19:25:04,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741915_1093 (size=61) 2024-12-04T19:25:04,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741915_1093 (size=61) 2024-12-04T19:25:04,153 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:25:04,154 INFO [StoreOpener-042e7b11f21e089c66c76494e91abf49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,156 INFO [StoreOpener-042e7b11f21e089c66c76494e91abf49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 042e7b11f21e089c66c76494e91abf49 columnFamilyName a 2024-12-04T19:25:04,156 DEBUG [StoreOpener-042e7b11f21e089c66c76494e91abf49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:04,156 INFO [StoreOpener-042e7b11f21e089c66c76494e91abf49-1 {}] regionserver.HStore(327): Store=042e7b11f21e089c66c76494e91abf49/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:04,157 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,157 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,158 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,158 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,158 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,160 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,162 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:25:04,163 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 042e7b11f21e089c66c76494e91abf49; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70217906, jitterRate=0.046328336000442505}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T19:25:04,164 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 042e7b11f21e089c66c76494e91abf49: Writing region info on filesystem at 1733340304153Initializing all the Stores at 1733340304154 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340304154Cleaning up temporary data from old regions at 1733340304158 (+4 ms)Region opened successfully at 1733340304164 (+6 ms) 2024-12-04T19:25:04,164 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 042e7b11f21e089c66c76494e91abf49, disabling compactions & flushes 2024-12-04T19:25:04,164 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49. 2024-12-04T19:25:04,164 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49. 2024-12-04T19:25:04,164 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49. after waiting 0 ms 2024-12-04T19:25:04,164 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49. 2024-12-04T19:25:04,164 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49. 2024-12-04T19:25:04,164 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 042e7b11f21e089c66c76494e91abf49: Waiting for close lock at 1733340304164Disabling compacts and flushes for region at 1733340304164Disabling writes for close at 1733340304164Writing region close event to WAL at 1733340304164Closed at 1733340304164 2024-12-04T19:25:04,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741914_1092 (size=175) 2024-12-04T19:25:04,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741914_1092 (size=175) 2024-12-04T19:25:04,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741914_1092 (size=175) 2024-12-04T19:25:04,169 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:25:04,169 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-79153010:(num 1733340304100) 2024-12-04T19:25:04,169 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:25:04,171 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340304094, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:25:04,182 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733340304094/wal.1733340304171, exclude list is [], retry=0 2024-12-04T19:25:04,185 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:25:04,185 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:25:04,186 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:25:04,206 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733340304094/wal.1733340304171 2024-12-04T19:25:04,207 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:33421:33421)] 2024-12-04T19:25:04,207 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 042e7b11f21e089c66c76494e91abf49, NAME => 'testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:25:04,207 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:25:04,207 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,207 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,208 INFO [StoreOpener-042e7b11f21e089c66c76494e91abf49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,209 INFO [StoreOpener-042e7b11f21e089c66c76494e91abf49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 042e7b11f21e089c66c76494e91abf49 columnFamilyName a 2024-12-04T19:25:04,209 DEBUG [StoreOpener-042e7b11f21e089c66c76494e91abf49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:04,210 INFO [StoreOpener-042e7b11f21e089c66c76494e91abf49-1 {}] regionserver.HStore(327): Store=042e7b11f21e089c66c76494e91abf49/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:04,210 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,210 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,211 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,212 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,212 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,213 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,214 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 042e7b11f21e089c66c76494e91abf49; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61775213, jitterRate=-0.07947759330272675}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T19:25:04,215 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 042e7b11f21e089c66c76494e91abf49: Writing region info on filesystem at 1733340304207Initializing all the Stores at 1733340304208 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340304208Cleaning up temporary data from old regions at 1733340304212 (+4 ms)Region opened successfully at 1733340304215 (+3 ms) 2024-12-04T19:25:04,223 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 042e7b11f21e089c66c76494e91abf49, disabling compactions & flushes 2024-12-04T19:25:04,223 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49. 2024-12-04T19:25:04,223 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49. 2024-12-04T19:25:04,224 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49. after waiting 0 ms 2024-12-04T19:25:04,224 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49. 2024-12-04T19:25:04,224 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49. 2024-12-04T19:25:04,224 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49. 2024-12-04T19:25:04,224 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 042e7b11f21e089c66c76494e91abf49: Waiting for close lock at 1733340304223Disabling compacts and flushes for region at 1733340304223Disabling writes for close at 1733340304224 (+1 ms)Writing region close event to WAL at 1733340304224Closed at 1733340304224 2024-12-04T19:25:04,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741916_1094 (size=1795) 2024-12-04T19:25:04,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741916_1094 (size=1795) 2024-12-04T19:25:04,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741916_1094 (size=1795) 2024-12-04T19:25:04,243 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33439/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340304094/wal.1733340304171, size=1.8 K (1795bytes) 2024-12-04T19:25:04,243 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33439/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340304094/wal.1733340304171 2024-12-04T19:25:04,243 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33439/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340304094/wal.1733340304171 after 0ms 2024-12-04T19:25:04,266 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:25:04,266 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33439/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340304094/wal.1733340304171 took 23ms 2024-12-04T19:25:04,268 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33439/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340304094/wal.1733340304171 so closing down 2024-12-04T19:25:04,268 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:25:04,269 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733340304171.temp 2024-12-04T19:25:04,291 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/recovered.edits/0000000000000000003-wal.1733340304171.temp 2024-12-04T19:25:04,292 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:25:04,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741917_1095 (size=1795) 2024-12-04T19:25:04,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741917_1095 (size=1795) 2024-12-04T19:25:04,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741917_1095 (size=1795) 2024-12-04T19:25:04,298 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/recovered.edits/0000000000000000003-wal.1733340304171.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-04T19:25:04,299 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/recovered.edits/0000000000000000003-wal.1733340304171.temp to hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/recovered.edits/0000000000000000012 2024-12-04T19:25:04,299 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 33 ms; skipped=0; WAL=hdfs://localhost:33439/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340304094/wal.1733340304171, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-12-04T19:25:04,299 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33439/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340304094/wal.1733340304171, journal: Splitting hdfs://localhost:33439/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340304094/wal.1733340304171, size=1.8 K (1795bytes) at 1733340304243Finishing writing output for hdfs://localhost:33439/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340304094/wal.1733340304171 so closing down at 1733340304268 (+25 ms)Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/recovered.edits/0000000000000000003-wal.1733340304171.temp at 1733340304292 (+24 ms)3 split writer threads finished at 1733340304292Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/recovered.edits/0000000000000000003-wal.1733340304171.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733340304298 (+6 ms)Rename recovered edits hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/recovered.edits/0000000000000000003-wal.1733340304171.temp to hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/recovered.edits/0000000000000000012 at 1733340304299 (+1 ms)Processed 10 edits across 1 Regions in 33 ms; skipped=0; WAL=hdfs://localhost:33439/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340304094/wal.1733340304171, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1733340304299 2024-12-04T19:25:04,301 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33439/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340304094/wal.1733340304171 to hdfs://localhost:33439/hbase/oldWALs/wal.1733340304171 2024-12-04T19:25:04,302 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/recovered.edits/0000000000000000012 2024-12-04T19:25:04,328 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:25:04,653 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:25:04,655 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/testdatalosswheninputerror-manual,16010,1733340304094, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:25:04,665 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733340304094/wal.1733340304656, exclude list is [], retry=0 2024-12-04T19:25:04,668 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:25:04,668 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:25:04,668 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:25:04,689 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733340304094/wal.1733340304656 2024-12-04T19:25:04,690 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:42335:42335)] 2024-12-04T19:25:04,690 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 042e7b11f21e089c66c76494e91abf49, NAME => 'testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:25:04,690 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:25:04,690 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,690 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,693 INFO [StoreOpener-042e7b11f21e089c66c76494e91abf49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,694 INFO [StoreOpener-042e7b11f21e089c66c76494e91abf49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 042e7b11f21e089c66c76494e91abf49 columnFamilyName a 2024-12-04T19:25:04,694 DEBUG [StoreOpener-042e7b11f21e089c66c76494e91abf49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:04,695 INFO [StoreOpener-042e7b11f21e089c66c76494e91abf49-1 {}] regionserver.HStore(327): Store=042e7b11f21e089c66c76494e91abf49/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:04,695 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,696 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,697 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,698 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/recovered.edits/0000000000000000012 2024-12-04T19:25:04,723 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:25:04,724 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/recovered.edits/0000000000000000012 2024-12-04T19:25:04,724 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 042e7b11f21e089c66c76494e91abf49 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-04T19:25:04,738 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/.tmp/a/1495ce9023ab41009f3267279fb06a7d is 79, key is testDatalossWhenInputError/a:x0/1733340304215/Put/seqid=0 2024-12-04T19:25:04,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741919_1097 (size=5808) 2024-12-04T19:25:04,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741919_1097 (size=5808) 2024-12-04T19:25:04,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741919_1097 (size=5808) 2024-12-04T19:25:04,744 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/.tmp/a/1495ce9023ab41009f3267279fb06a7d 2024-12-04T19:25:04,753 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/.tmp/a/1495ce9023ab41009f3267279fb06a7d as hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/a/1495ce9023ab41009f3267279fb06a7d 2024-12-04T19:25:04,762 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/a/1495ce9023ab41009f3267279fb06a7d, entries=10, sequenceid=12, filesize=5.7 K 2024-12-04T19:25:04,763 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 042e7b11f21e089c66c76494e91abf49 in 39ms, sequenceid=12, compaction requested=false; wal=null 2024-12-04T19:25:04,763 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/recovered.edits/0000000000000000012 2024-12-04T19:25:04,765 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,765 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,768 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,770 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-04T19:25:04,770 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 042e7b11f21e089c66c76494e91abf49; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58952687, jitterRate=-0.1215365082025528}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T19:25:04,771 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 042e7b11f21e089c66c76494e91abf49: Writing region info on filesystem at 1733340304690Initializing all the Stores at 1733340304692 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340304692Obtaining lock to block concurrent updates at 1733340304724 (+32 ms)Preparing flush snapshotting stores in 042e7b11f21e089c66c76494e91abf49 at 1733340304724Finished memstore snapshotting testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733340304724Flushing stores of testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49. at 1733340304724Flushing 042e7b11f21e089c66c76494e91abf49/a: creating writer at 1733340304724Flushing 042e7b11f21e089c66c76494e91abf49/a: appending metadata at 1733340304738 (+14 ms)Flushing 042e7b11f21e089c66c76494e91abf49/a: closing flushed file at 1733340304738Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43ad1b27: reopening flushed file at 1733340304751 (+13 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 042e7b11f21e089c66c76494e91abf49 in 39ms, sequenceid=12, compaction requested=false; wal=null at 1733340304763 (+12 ms)Cleaning up temporary data from old regions at 1733340304765 (+2 ms)Region opened successfully at 1733340304771 (+6 ms) 2024-12-04T19:25:04,773 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 042e7b11f21e089c66c76494e91abf49, NAME => 'testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:25:04,774 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733340304095.042e7b11f21e089c66c76494e91abf49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:25:04,774 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,774 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,775 INFO [StoreOpener-042e7b11f21e089c66c76494e91abf49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,775 INFO [StoreOpener-042e7b11f21e089c66c76494e91abf49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 042e7b11f21e089c66c76494e91abf49 columnFamilyName a 2024-12-04T19:25:04,775 DEBUG [StoreOpener-042e7b11f21e089c66c76494e91abf49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:04,781 DEBUG [StoreOpener-042e7b11f21e089c66c76494e91abf49-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/a/1495ce9023ab41009f3267279fb06a7d 2024-12-04T19:25:04,781 INFO [StoreOpener-042e7b11f21e089c66c76494e91abf49-1 {}] regionserver.HStore(327): Store=042e7b11f21e089c66c76494e91abf49/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:04,781 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,782 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,783 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,784 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,784 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,786 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 042e7b11f21e089c66c76494e91abf49 2024-12-04T19:25:04,788 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testDatalossWhenInputError/042e7b11f21e089c66c76494e91abf49/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-04T19:25:04,789 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 042e7b11f21e089c66c76494e91abf49; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63850011, jitterRate=-0.048560693860054016}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T19:25:04,789 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 042e7b11f21e089c66c76494e91abf49: Writing region info on filesystem at 1733340304774Initializing all the Stores at 1733340304774Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340304774Cleaning up temporary data from old regions at 1733340304784 (+10 ms)Region opened successfully at 1733340304789 (+5 ms) 2024-12-04T19:25:04,809 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=425 (was 415) Potentially hanging thread: PacketResponder: BP-1762372386-172.17.0.2-1733340268509:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:42780 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:42872 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:60982 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1762372386-172.17.0.2-1733340268509:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:54658 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:54718 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1762372386-172.17.0.2-1733340268509:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:32800 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1262 (was 1180) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=144 (was 148), ProcessCount=11 (was 11), AvailableMemoryMB=15528 (was 15536) 2024-12-04T19:25:04,809 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-12-04T19:25:04,826 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=425, OpenFileDescriptor=1262, MaxFileDescriptor=1048576, SystemLoadAverage=144, ProcessCount=11, AvailableMemoryMB=15527 2024-12-04T19:25:04,826 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1262 is superior to 1024 2024-12-04T19:25:04,843 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:25:04,845 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:25:04,846 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:25:04,848 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-63004807, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/hregion-63004807, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:25:04,863 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-63004807/hregion-63004807.1733340304849, exclude list is [], retry=0 2024-12-04T19:25:04,866 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:25:04,866 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:25:04,866 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:25:04,896 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-63004807/hregion-63004807.1733340304849 2024-12-04T19:25:04,897 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:37675:37675)] 2024-12-04T19:25:04,897 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 28737fab5e8654ff32b5aaf9e3ec3bc3, NAME => 'testCompactedBulkLoadedFiles,,1733340304844.28737fab5e8654ff32b5aaf9e3ec3bc3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33439/hbase 2024-12-04T19:25:04,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741921_1099 (size=63) 2024-12-04T19:25:04,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741921_1099 (size=63) 2024-12-04T19:25:04,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741921_1099 (size=63) 2024-12-04T19:25:04,906 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733340304844.28737fab5e8654ff32b5aaf9e3ec3bc3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:25:04,907 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:04,908 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28737fab5e8654ff32b5aaf9e3ec3bc3 columnFamilyName a 2024-12-04T19:25:04,908 DEBUG [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:04,909 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.HStore(327): Store=28737fab5e8654ff32b5aaf9e3ec3bc3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:04,909 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:04,910 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28737fab5e8654ff32b5aaf9e3ec3bc3 columnFamilyName b 2024-12-04T19:25:04,910 DEBUG [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:04,910 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.HStore(327): Store=28737fab5e8654ff32b5aaf9e3ec3bc3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:04,910 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:04,911 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28737fab5e8654ff32b5aaf9e3ec3bc3 columnFamilyName c 2024-12-04T19:25:04,911 DEBUG [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:04,912 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.HStore(327): Store=28737fab5e8654ff32b5aaf9e3ec3bc3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:04,912 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:04,913 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:04,913 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:04,914 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:04,914 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:04,915 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:25:04,916 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:04,918 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:25:04,918 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28737fab5e8654ff32b5aaf9e3ec3bc3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67409713, jitterRate=0.0044829994440078735}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:25:04,919 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28737fab5e8654ff32b5aaf9e3ec3bc3: Writing region info on filesystem at 1733340304906Initializing all the Stores at 1733340304907 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340304907Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340304907Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340304907Cleaning up temporary data from old regions at 1733340304914 (+7 ms)Region opened successfully at 1733340304919 (+5 ms) 2024-12-04T19:25:04,919 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 28737fab5e8654ff32b5aaf9e3ec3bc3, disabling compactions & flushes 2024-12-04T19:25:04,919 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733340304844.28737fab5e8654ff32b5aaf9e3ec3bc3. 2024-12-04T19:25:04,919 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733340304844.28737fab5e8654ff32b5aaf9e3ec3bc3. 2024-12-04T19:25:04,919 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733340304844.28737fab5e8654ff32b5aaf9e3ec3bc3. after waiting 0 ms 2024-12-04T19:25:04,919 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733340304844.28737fab5e8654ff32b5aaf9e3ec3bc3. 2024-12-04T19:25:04,919 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733340304844.28737fab5e8654ff32b5aaf9e3ec3bc3. 2024-12-04T19:25:04,919 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 28737fab5e8654ff32b5aaf9e3ec3bc3: Waiting for close lock at 1733340304919Disabling compacts and flushes for region at 1733340304919Disabling writes for close at 1733340304919Writing region close event to WAL at 1733340304919Closed at 1733340304919 2024-12-04T19:25:04,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741920_1098 (size=175) 2024-12-04T19:25:04,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741920_1098 (size=175) 2024-12-04T19:25:04,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741920_1098 (size=175) 2024-12-04T19:25:04,924 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:25:04,924 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-63004807:(num 1733340304849) 2024-12-04T19:25:04,924 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:25:04,925 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:25:04,938 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843/wal.1733340304925, exclude list is [], retry=0 2024-12-04T19:25:04,940 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:25:04,940 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:25:04,941 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:25:04,963 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843/wal.1733340304925 2024-12-04T19:25:04,964 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:42335:42335)] 2024-12-04T19:25:04,964 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 28737fab5e8654ff32b5aaf9e3ec3bc3, NAME => 'testCompactedBulkLoadedFiles,,1733340304844.28737fab5e8654ff32b5aaf9e3ec3bc3.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:25:04,964 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733340304844.28737fab5e8654ff32b5aaf9e3ec3bc3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:25:04,964 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:04,964 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:04,965 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:04,966 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28737fab5e8654ff32b5aaf9e3ec3bc3 columnFamilyName a 2024-12-04T19:25:04,966 DEBUG [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:04,966 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.HStore(327): Store=28737fab5e8654ff32b5aaf9e3ec3bc3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:04,966 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:04,967 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28737fab5e8654ff32b5aaf9e3ec3bc3 columnFamilyName b 2024-12-04T19:25:04,967 DEBUG [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:04,968 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.HStore(327): Store=28737fab5e8654ff32b5aaf9e3ec3bc3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:04,968 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:04,969 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28737fab5e8654ff32b5aaf9e3ec3bc3 columnFamilyName c 2024-12-04T19:25:04,969 DEBUG [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:04,969 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.HStore(327): Store=28737fab5e8654ff32b5aaf9e3ec3bc3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:04,969 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:04,970 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:04,971 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:04,972 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:04,972 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:04,972 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:25:04,973 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:04,974 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28737fab5e8654ff32b5aaf9e3ec3bc3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70711013, jitterRate=0.05367620289325714}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:25:04,974 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28737fab5e8654ff32b5aaf9e3ec3bc3: Writing region info on filesystem at 1733340304964Initializing all the Stores at 1733340304965 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340304965Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340304965Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340304965Cleaning up temporary data from old regions at 1733340304972 (+7 ms)Region opened successfully at 1733340304974 (+2 ms) 2024-12-04T19:25:04,978 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733340304977/Put/seqid=0 2024-12-04T19:25:04,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741923_1101 (size=4875) 2024-12-04T19:25:04,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741923_1101 (size=4875) 2024-12-04T19:25:04,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741923_1101 (size=4875) 2024-12-04T19:25:04,987 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733340304987/Put/seqid=0 2024-12-04T19:25:04,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741924_1102 (size=4875) 2024-12-04T19:25:04,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741924_1102 (size=4875) 2024-12-04T19:25:04,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741924_1102 (size=4875) 2024-12-04T19:25:04,995 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733340304995/Put/seqid=0 2024-12-04T19:25:05,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741925_1103 (size=4875) 2024-12-04T19:25:05,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741925_1103 (size=4875) 2024-12-04T19:25:05,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741925_1103 (size=4875) 2024-12-04T19:25:05,004 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33439/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 28737fab5e8654ff32b5aaf9e3ec3bc3/a 2024-12-04T19:25:05,007 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-04T19:25:05,007 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T19:25:05,007 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33439/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 28737fab5e8654ff32b5aaf9e3ec3bc3/a 2024-12-04T19:25:05,010 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-04T19:25:05,010 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T19:25:05,010 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33439/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 28737fab5e8654ff32b5aaf9e3ec3bc3/a 2024-12-04T19:25:05,013 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-04T19:25:05,013 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T19:25:05,013 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 28737fab5e8654ff32b5aaf9e3ec3bc3 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-04T19:25:05,026 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/.tmp/a/8a5a01b749d04568a0c1f168a10b176c is 55, key is testCompactedBulkLoadedFiles/a:a/1733340304975/Put/seqid=0 2024-12-04T19:25:05,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741926_1104 (size=5107) 2024-12-04T19:25:05,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741926_1104 (size=5107) 2024-12-04T19:25:05,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741926_1104 (size=5107) 2024-12-04T19:25:05,032 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/.tmp/a/8a5a01b749d04568a0c1f168a10b176c 2024-12-04T19:25:05,038 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/.tmp/a/8a5a01b749d04568a0c1f168a10b176c as hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/8a5a01b749d04568a0c1f168a10b176c 2024-12-04T19:25:05,044 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/8a5a01b749d04568a0c1f168a10b176c, entries=1, sequenceid=4, filesize=5.0 K 2024-12-04T19:25:05,046 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 28737fab5e8654ff32b5aaf9e3ec3bc3 in 33ms, sequenceid=4, compaction requested=false 2024-12-04T19:25:05,046 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 28737fab5e8654ff32b5aaf9e3ec3bc3: 2024-12-04T19:25:05,047 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/94d15618bb3f4109903db8e448a83c73_SeqId_4_ 2024-12-04T19:25:05,048 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/80936c31467d49219c7eed18eb3723fa_SeqId_4_ 2024-12-04T19:25:05,049 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/630597735b1848a8a7116e9fe9f6ff6a_SeqId_4_ 2024-12-04T19:25:05,049 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33439/hbase/testCompactedBulkLoadedFiles/hfile0 into 28737fab5e8654ff32b5aaf9e3ec3bc3/a as hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/94d15618bb3f4109903db8e448a83c73_SeqId_4_ - updating store file list. 2024-12-04T19:25:05,053 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 94d15618bb3f4109903db8e448a83c73_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T19:25:05,053 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/94d15618bb3f4109903db8e448a83c73_SeqId_4_ into 28737fab5e8654ff32b5aaf9e3ec3bc3/a 2024-12-04T19:25:05,053 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33439/hbase/testCompactedBulkLoadedFiles/hfile0 into 28737fab5e8654ff32b5aaf9e3ec3bc3/a (new location: hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/94d15618bb3f4109903db8e448a83c73_SeqId_4_) 2024-12-04T19:25:05,054 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33439/hbase/testCompactedBulkLoadedFiles/hfile1 into 28737fab5e8654ff32b5aaf9e3ec3bc3/a as hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/80936c31467d49219c7eed18eb3723fa_SeqId_4_ - updating store file list. 2024-12-04T19:25:05,059 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 80936c31467d49219c7eed18eb3723fa_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T19:25:05,059 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/80936c31467d49219c7eed18eb3723fa_SeqId_4_ into 28737fab5e8654ff32b5aaf9e3ec3bc3/a 2024-12-04T19:25:05,059 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33439/hbase/testCompactedBulkLoadedFiles/hfile1 into 28737fab5e8654ff32b5aaf9e3ec3bc3/a (new location: hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/80936c31467d49219c7eed18eb3723fa_SeqId_4_) 2024-12-04T19:25:05,060 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33439/hbase/testCompactedBulkLoadedFiles/hfile2 into 28737fab5e8654ff32b5aaf9e3ec3bc3/a as hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/630597735b1848a8a7116e9fe9f6ff6a_SeqId_4_ - updating store file list. 2024-12-04T19:25:05,065 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 630597735b1848a8a7116e9fe9f6ff6a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T19:25:05,065 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/630597735b1848a8a7116e9fe9f6ff6a_SeqId_4_ into 28737fab5e8654ff32b5aaf9e3ec3bc3/a 2024-12-04T19:25:05,065 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33439/hbase/testCompactedBulkLoadedFiles/hfile2 into 28737fab5e8654ff32b5aaf9e3ec3bc3/a (new location: hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/630597735b1848a8a7116e9fe9f6ff6a_SeqId_4_) 2024-12-04T19:25:05,073 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T19:25:05,073 DEBUG [Time-limited test {}] regionserver.HStore(1541): 28737fab5e8654ff32b5aaf9e3ec3bc3/a is initiating major compaction (all files) 2024-12-04T19:25:05,073 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 28737fab5e8654ff32b5aaf9e3ec3bc3/a in testCompactedBulkLoadedFiles,,1733340304844.28737fab5e8654ff32b5aaf9e3ec3bc3. 2024-12-04T19:25:05,073 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/8a5a01b749d04568a0c1f168a10b176c, hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/94d15618bb3f4109903db8e448a83c73_SeqId_4_, hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/80936c31467d49219c7eed18eb3723fa_SeqId_4_, hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/630597735b1848a8a7116e9fe9f6ff6a_SeqId_4_] into tmpdir=hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/.tmp, totalSize=19.3 K 2024-12-04T19:25:05,074 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8a5a01b749d04568a0c1f168a10b176c, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733340304975 2024-12-04T19:25:05,074 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 94d15618bb3f4109903db8e448a83c73_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-04T19:25:05,075 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 80936c31467d49219c7eed18eb3723fa_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-04T19:25:05,075 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 630597735b1848a8a7116e9fe9f6ff6a_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-04T19:25:05,087 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/.tmp/a/8bb2335a4214418097b59ee110db0f65 is 55, key is testCompactedBulkLoadedFiles/a:a/1733340304975/Put/seqid=0 2024-12-04T19:25:05,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741927_1105 (size=6154) 2024-12-04T19:25:05,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741927_1105 (size=6154) 2024-12-04T19:25:05,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741927_1105 (size=6154) 2024-12-04T19:25:05,100 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/.tmp/a/8bb2335a4214418097b59ee110db0f65 as hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/8bb2335a4214418097b59ee110db0f65 2024-12-04T19:25:05,107 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 28737fab5e8654ff32b5aaf9e3ec3bc3/a of 28737fab5e8654ff32b5aaf9e3ec3bc3 into 8bb2335a4214418097b59ee110db0f65(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T19:25:05,107 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 28737fab5e8654ff32b5aaf9e3ec3bc3: 2024-12-04T19:25:05,107 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-04T19:25:05,107 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-04T19:25:05,136 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33439/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843/wal.1733340304925, size=0 (0bytes) 2024-12-04T19:25:05,136 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33439/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843/wal.1733340304925 might be still open, length is 0 2024-12-04T19:25:05,136 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33439/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843/wal.1733340304925 2024-12-04T19:25:05,137 WARN [IPC Server handler 2 on default port 33439 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843/wal.1733340304925 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-12-04T19:25:05,137 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33439/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843/wal.1733340304925 after 1ms 2024-12-04T19:25:07,816 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:32834 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:44885:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:32834 dst: /127.0.0.1:44885 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:44885 remote=/127.0.0.1:32834]. Total timeout mills is 60000, 57291 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:25:07,816 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:54752 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:41199:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54752 dst: /127.0.0.1:41199 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:25:07,816 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:42908 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:38107:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42908 dst: /127.0.0.1:38107 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:25:07,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741922_1106 (size=1794) 2024-12-04T19:25:07,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741922_1106 (size=1794) 2024-12-04T19:25:07,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741922_1106 (size=1794) 2024-12-04T19:25:09,138 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33439/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843/wal.1733340304925 after 4002ms 2024-12-04T19:25:09,167 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:25:09,167 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33439/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843/wal.1733340304925 took 4031ms 2024-12-04T19:25:09,169 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33439/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843/wal.1733340304925; continuing. 2024-12-04T19:25:09,169 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33439/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843/wal.1733340304925 so closing down 2024-12-04T19:25:09,169 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:25:09,171 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733340304925.temp 2024-12-04T19:25:09,193 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/recovered.edits/0000000000000000003-wal.1733340304925.temp 2024-12-04T19:25:09,193 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:25:09,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741928_1107 (size=800) 2024-12-04T19:25:09,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741928_1107 (size=800) 2024-12-04T19:25:09,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741928_1107 (size=800) 2024-12-04T19:25:09,199 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/recovered.edits/0000000000000000003-wal.1733340304925.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-04T19:25:09,201 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/recovered.edits/0000000000000000003-wal.1733340304925.temp to hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/recovered.edits/0000000000000000008 2024-12-04T19:25:09,201 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 33 ms; skipped=3; WAL=hdfs://localhost:33439/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843/wal.1733340304925, size=0, length=0, corrupted=false, cancelled=false 2024-12-04T19:25:09,201 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33439/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843/wal.1733340304925, journal: Splitting hdfs://localhost:33439/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843/wal.1733340304925, size=0 (0bytes) at 1733340305136Finishing writing output for hdfs://localhost:33439/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843/wal.1733340304925 so closing down at 1733340309169 (+4033 ms)Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/recovered.edits/0000000000000000003-wal.1733340304925.temp at 1733340309193 (+24 ms)3 split writer threads finished at 1733340309193Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/recovered.edits/0000000000000000003-wal.1733340304925.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733340309199 (+6 ms)Rename recovered edits hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/recovered.edits/0000000000000000003-wal.1733340304925.temp to hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/recovered.edits/0000000000000000008 at 1733340309201 (+2 ms)Processed 5 edits across 1 Regions in 33 ms; skipped=3; WAL=hdfs://localhost:33439/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843/wal.1733340304925, size=0, length=0, corrupted=false, cancelled=false at 1733340309201 2024-12-04T19:25:09,202 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33439/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843/wal.1733340304925 to hdfs://localhost:33439/hbase/oldWALs/wal.1733340304925 2024-12-04T19:25:09,203 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/recovered.edits/0000000000000000008 2024-12-04T19:25:09,203 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:25:09,204 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:25:09,216 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843/wal.1733340309205, exclude list is [], retry=0 2024-12-04T19:25:09,218 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:25:09,218 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:25:09,219 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:25:09,239 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843/wal.1733340309205 2024-12-04T19:25:09,240 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:42335:42335)] 2024-12-04T19:25:09,240 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 28737fab5e8654ff32b5aaf9e3ec3bc3, NAME => 'testCompactedBulkLoadedFiles,,1733340304844.28737fab5e8654ff32b5aaf9e3ec3bc3.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:25:09,240 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733340304844.28737fab5e8654ff32b5aaf9e3ec3bc3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:25:09,240 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:09,240 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:09,241 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:09,242 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28737fab5e8654ff32b5aaf9e3ec3bc3 columnFamilyName a 2024-12-04T19:25:09,242 DEBUG [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:09,248 DEBUG [StoreFileOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 630597735b1848a8a7116e9fe9f6ff6a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T19:25:09,249 DEBUG [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/630597735b1848a8a7116e9fe9f6ff6a_SeqId_4_ 2024-12-04T19:25:09,252 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T19:25:09,252 DEBUG [StoreFileOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 80936c31467d49219c7eed18eb3723fa_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T19:25:09,252 DEBUG [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/80936c31467d49219c7eed18eb3723fa_SeqId_4_ 2024-12-04T19:25:09,256 DEBUG [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/8a5a01b749d04568a0c1f168a10b176c 2024-12-04T19:25:09,261 DEBUG [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/8bb2335a4214418097b59ee110db0f65 2024-12-04T19:25:09,265 DEBUG [StoreFileOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 94d15618bb3f4109903db8e448a83c73_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T19:25:09,265 DEBUG [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/94d15618bb3f4109903db8e448a83c73_SeqId_4_ 2024-12-04T19:25:09,265 WARN [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/630597735b1848a8a7116e9fe9f6ff6a_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@9dfe152 2024-12-04T19:25:09,265 WARN [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/80936c31467d49219c7eed18eb3723fa_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@9dfe152 2024-12-04T19:25:09,265 WARN [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/8a5a01b749d04568a0c1f168a10b176c from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@9dfe152 2024-12-04T19:25:09,265 WARN [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/94d15618bb3f4109903db8e448a83c73_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@9dfe152 2024-12-04T19:25:09,266 DEBUG [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/630597735b1848a8a7116e9fe9f6ff6a_SeqId_4_, hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/80936c31467d49219c7eed18eb3723fa_SeqId_4_, hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/8a5a01b749d04568a0c1f168a10b176c, hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/94d15618bb3f4109903db8e448a83c73_SeqId_4_] to archive 2024-12-04T19:25:09,267 DEBUG [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T19:25:09,268 DEBUG [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/630597735b1848a8a7116e9fe9f6ff6a_SeqId_4_ to hdfs://localhost:33439/hbase/archive/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/630597735b1848a8a7116e9fe9f6ff6a_SeqId_4_ 2024-12-04T19:25:09,270 DEBUG [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/80936c31467d49219c7eed18eb3723fa_SeqId_4_ to hdfs://localhost:33439/hbase/archive/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/80936c31467d49219c7eed18eb3723fa_SeqId_4_ 2024-12-04T19:25:09,271 DEBUG [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/8a5a01b749d04568a0c1f168a10b176c to hdfs://localhost:33439/hbase/archive/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/8a5a01b749d04568a0c1f168a10b176c 2024-12-04T19:25:09,272 DEBUG [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/94d15618bb3f4109903db8e448a83c73_SeqId_4_ to hdfs://localhost:33439/hbase/archive/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/a/94d15618bb3f4109903db8e448a83c73_SeqId_4_ 2024-12-04T19:25:09,273 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.HStore(327): Store=28737fab5e8654ff32b5aaf9e3ec3bc3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:09,273 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:09,273 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28737fab5e8654ff32b5aaf9e3ec3bc3 columnFamilyName b 2024-12-04T19:25:09,274 DEBUG [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:09,274 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.HStore(327): Store=28737fab5e8654ff32b5aaf9e3ec3bc3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:09,274 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:09,275 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 28737fab5e8654ff32b5aaf9e3ec3bc3 columnFamilyName c 2024-12-04T19:25:09,275 DEBUG [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:09,275 INFO [StoreOpener-28737fab5e8654ff32b5aaf9e3ec3bc3-1 {}] regionserver.HStore(327): Store=28737fab5e8654ff32b5aaf9e3ec3bc3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:09,275 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:09,276 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:09,277 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:09,278 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/recovered.edits/0000000000000000008 2024-12-04T19:25:09,300 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:25:09,302 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 28737fab5e8654ff32b5aaf9e3ec3bc3 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "28737fab5e8654ff32b5aaf9e3ec3bc3" family_name: "a" compaction_input: "8a5a01b749d04568a0c1f168a10b176c" compaction_input: "94d15618bb3f4109903db8e448a83c73_SeqId_4_" compaction_input: "80936c31467d49219c7eed18eb3723fa_SeqId_4_" compaction_input: "630597735b1848a8a7116e9fe9f6ff6a_SeqId_4_" compaction_output: "8bb2335a4214418097b59ee110db0f65" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733340304844.28737fab5e8654ff32b5aaf9e3ec3bc3." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-04T19:25:09,302 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-04T19:25:09,302 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/recovered.edits/0000000000000000008 2024-12-04T19:25:09,303 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/recovered.edits/0000000000000000008 2024-12-04T19:25:09,304 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:09,304 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:09,305 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:25:09,306 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 28737fab5e8654ff32b5aaf9e3ec3bc3 2024-12-04T19:25:09,308 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testCompactedBulkLoadedFiles/28737fab5e8654ff32b5aaf9e3ec3bc3/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-04T19:25:09,309 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 28737fab5e8654ff32b5aaf9e3ec3bc3; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72696091, jitterRate=0.08325617015361786}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:25:09,309 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 28737fab5e8654ff32b5aaf9e3ec3bc3: Writing region info on filesystem at 1733340309240Initializing all the Stores at 1733340309241 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340309241Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340309241Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340309241Cleaning up temporary data from old regions at 1733340309304 (+63 ms)Region opened successfully at 1733340309309 (+5 ms) 2024-12-04T19:25:09,311 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 28737fab5e8654ff32b5aaf9e3ec3bc3, disabling compactions & flushes 2024-12-04T19:25:09,311 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733340304844.28737fab5e8654ff32b5aaf9e3ec3bc3. 2024-12-04T19:25:09,312 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733340304844.28737fab5e8654ff32b5aaf9e3ec3bc3. 2024-12-04T19:25:09,312 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733340304844.28737fab5e8654ff32b5aaf9e3ec3bc3. after waiting 0 ms 2024-12-04T19:25:09,312 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733340304844.28737fab5e8654ff32b5aaf9e3ec3bc3. 2024-12-04T19:25:09,312 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733340304844.28737fab5e8654ff32b5aaf9e3ec3bc3. 2024-12-04T19:25:09,312 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 28737fab5e8654ff32b5aaf9e3ec3bc3: Waiting for close lock at 1733340309311Disabling compacts and flushes for region at 1733340309311Disabling writes for close at 1733340309312 (+1 ms)Writing region close event to WAL at 1733340309312Closed at 1733340309312 2024-12-04T19:25:09,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741929_1108 (size=175) 2024-12-04T19:25:09,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741929_1108 (size=175) 2024-12-04T19:25:09,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741929_1108 (size=175) 2024-12-04T19:25:09,316 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:25:09,316 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733340309205) 2024-12-04T19:25:09,329 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=438 (was 425) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_62047591_22 at /127.0.0.1:54794 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:33439 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:33439 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_62047591_22 at /127.0.0.1:42996 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1343 (was 1262) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=181 (was 144) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=15510 (was 15527) 2024-12-04T19:25:09,329 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1343 is superior to 1024 2024-12-04T19:25:09,340 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=438, OpenFileDescriptor=1343, MaxFileDescriptor=1048576, SystemLoadAverage=181, ProcessCount=11, AvailableMemoryMB=15509 2024-12-04T19:25:09,340 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1343 is superior to 1024 2024-12-04T19:25:09,351 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:25:09,353 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T19:25:09,354 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T19:25:09,356 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-23235920, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/hregion-23235920, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:25:09,367 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-23235920/hregion-23235920.1733340309357, exclude list is [], retry=0 2024-12-04T19:25:09,370 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:25:09,370 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:25:09,370 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:25:09,393 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-23235920/hregion-23235920.1733340309357 2024-12-04T19:25:09,393 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:37675:37675)] 2024-12-04T19:25:09,393 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c53807fc839cea0d2df99fd1d43cdf76, NAME => 'testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33439/hbase 2024-12-04T19:25:09,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741931_1110 (size=67) 2024-12-04T19:25:09,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741931_1110 (size=67) 2024-12-04T19:25:09,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741931_1110 (size=67) 2024-12-04T19:25:09,405 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:25:09,405 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:09,407 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c53807fc839cea0d2df99fd1d43cdf76 columnFamilyName a 2024-12-04T19:25:09,407 DEBUG [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:09,407 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(327): Store=c53807fc839cea0d2df99fd1d43cdf76/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:09,407 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:09,408 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c53807fc839cea0d2df99fd1d43cdf76 columnFamilyName b 2024-12-04T19:25:09,408 DEBUG [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:09,409 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(327): Store=c53807fc839cea0d2df99fd1d43cdf76/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:09,409 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:09,410 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c53807fc839cea0d2df99fd1d43cdf76 columnFamilyName c 2024-12-04T19:25:09,410 DEBUG [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:09,411 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(327): Store=c53807fc839cea0d2df99fd1d43cdf76/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:09,411 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:09,411 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:09,412 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:09,413 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:09,413 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:09,413 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:25:09,414 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:09,416 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T19:25:09,417 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c53807fc839cea0d2df99fd1d43cdf76; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60425760, jitterRate=-0.09958600997924805}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:25:09,417 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c53807fc839cea0d2df99fd1d43cdf76: Writing region info on filesystem at 1733340309405Initializing all the Stores at 1733340309405Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340309405Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340309405Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340309405Cleaning up temporary data from old regions at 1733340309413 (+8 ms)Region opened successfully at 1733340309417 (+4 ms) 2024-12-04T19:25:09,417 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c53807fc839cea0d2df99fd1d43cdf76, disabling compactions & flushes 2024-12-04T19:25:09,417 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76. 2024-12-04T19:25:09,417 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76. 2024-12-04T19:25:09,417 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76. after waiting 0 ms 2024-12-04T19:25:09,417 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76. 2024-12-04T19:25:09,417 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76. 2024-12-04T19:25:09,417 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c53807fc839cea0d2df99fd1d43cdf76: Waiting for close lock at 1733340309417Disabling compacts and flushes for region at 1733340309417Disabling writes for close at 1733340309417Writing region close event to WAL at 1733340309417Closed at 1733340309417 2024-12-04T19:25:09,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741930_1109 (size=175) 2024-12-04T19:25:09,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741930_1109 (size=175) 2024-12-04T19:25:09,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741930_1109 (size=175) 2024-12-04T19:25:09,422 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:25:09,422 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-23235920:(num 1733340309357) 2024-12-04T19:25:09,422 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:25:09,423 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:25:09,434 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340309423, exclude list is [], retry=0 2024-12-04T19:25:09,437 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:25:09,437 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:25:09,437 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:25:09,460 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340309423 2024-12-04T19:25:09,460 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33421:33421),(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:42335:42335)] 2024-12-04T19:25:09,461 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c53807fc839cea0d2df99fd1d43cdf76, NAME => 'testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:25:09,461 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:25:09,461 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:09,461 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:09,462 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:09,463 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c53807fc839cea0d2df99fd1d43cdf76 columnFamilyName a 2024-12-04T19:25:09,463 DEBUG [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:09,463 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(327): Store=c53807fc839cea0d2df99fd1d43cdf76/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:09,464 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:09,464 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c53807fc839cea0d2df99fd1d43cdf76 columnFamilyName b 2024-12-04T19:25:09,465 DEBUG [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:09,465 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(327): Store=c53807fc839cea0d2df99fd1d43cdf76/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:09,465 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:09,466 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c53807fc839cea0d2df99fd1d43cdf76 columnFamilyName c 2024-12-04T19:25:09,466 DEBUG [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:09,467 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(327): Store=c53807fc839cea0d2df99fd1d43cdf76/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:09,467 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:09,467 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:09,468 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:09,469 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:09,469 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:09,469 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:25:09,470 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:09,471 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c53807fc839cea0d2df99fd1d43cdf76; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67115852, jitterRate=1.0412931442260742E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:25:09,472 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c53807fc839cea0d2df99fd1d43cdf76: Writing region info on filesystem at 1733340309461Initializing all the Stores at 1733340309462 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340309462Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340309462Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340309462Cleaning up temporary data from old regions at 1733340309469 (+7 ms)Region opened successfully at 1733340309471 (+2 ms) 2024-12-04T19:25:09,479 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c53807fc839cea0d2df99fd1d43cdf76 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-04T19:25:09,493 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/.tmp/a/9451179a3b604ab892aaacc2967e3ec3 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733340309472/Put/seqid=0 2024-12-04T19:25:09,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741933_1112 (size=5958) 2024-12-04T19:25:09,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741933_1112 (size=5958) 2024-12-04T19:25:09,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741933_1112 (size=5958) 2024-12-04T19:25:09,900 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/.tmp/a/9451179a3b604ab892aaacc2967e3ec3 2024-12-04T19:25:09,910 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/.tmp/a/9451179a3b604ab892aaacc2967e3ec3 as hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/a/9451179a3b604ab892aaacc2967e3ec3 2024-12-04T19:25:09,917 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/a/9451179a3b604ab892aaacc2967e3ec3, entries=10, sequenceid=13, filesize=5.8 K 2024-12-04T19:25:09,919 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for c53807fc839cea0d2df99fd1d43cdf76 in 440ms, sequenceid=13, compaction requested=false 2024-12-04T19:25:09,919 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c53807fc839cea0d2df99fd1d43cdf76: 2024-12-04T19:25:09,937 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c53807fc839cea0d2df99fd1d43cdf76, disabling compactions & flushes 2024-12-04T19:25:09,937 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76. 2024-12-04T19:25:09,937 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76. 2024-12-04T19:25:09,938 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76. after waiting 1 ms 2024-12-04T19:25:09,938 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76. 2024-12-04T19:25:09,938 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76. 2024-12-04T19:25:09,938 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76. 2024-12-04T19:25:09,938 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c53807fc839cea0d2df99fd1d43cdf76: Waiting for close lock at 1733340309937Disabling compacts and flushes for region at 1733340309937Disabling writes for close at 1733340309938 (+1 ms)Writing region close event to WAL at 1733340309938Closed at 1733340309938 2024-12-04T19:25:09,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741932_1111 (size=6243) 2024-12-04T19:25:09,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741932_1111 (size=6243) 2024-12-04T19:25:09,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741932_1111 (size=6243) 2024-12-04T19:25:09,955 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340309423, size=6.1 K (6243bytes) 2024-12-04T19:25:09,955 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340309423 2024-12-04T19:25:09,955 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340309423 after 0ms 2024-12-04T19:25:09,977 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:25:09,977 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340309423 took 22ms 2024-12-04T19:25:09,979 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340309423 so closing down 2024-12-04T19:25:09,979 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:25:09,980 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733340309423.temp 2024-12-04T19:25:10,002 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000003-wal.1733340309423.temp 2024-12-04T19:25:10,002 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:25:10,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741934_1113 (size=5575) 2024-12-04T19:25:10,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741934_1113 (size=5575) 2024-12-04T19:25:10,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741934_1113 (size=5575) 2024-12-04T19:25:10,008 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000003-wal.1733340309423.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-04T19:25:10,009 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000003-wal.1733340309423.temp to hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000035 2024-12-04T19:25:10,010 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 31 ms; skipped=2; WAL=hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340309423, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-12-04T19:25:10,010 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340309423, journal: Splitting hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340309423, size=6.1 K (6243bytes) at 1733340309955Finishing writing output for hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340309423 so closing down at 1733340309979 (+24 ms)Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000003-wal.1733340309423.temp at 1733340310002 (+23 ms)3 split writer threads finished at 1733340310002Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000003-wal.1733340309423.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733340310008 (+6 ms)Rename recovered edits hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000003-wal.1733340309423.temp to hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000035 at 1733340310009 (+1 ms)Processed 32 edits across 1 Regions in 31 ms; skipped=2; WAL=hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340309423, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1733340310010 (+1 ms) 2024-12-04T19:25:10,011 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340309423 to hdfs://localhost:33439/hbase/oldWALs/wal.1733340309423 2024-12-04T19:25:10,012 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000035 2024-12-04T19:25:10,012 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:25:10,014 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:25:10,026 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340310014, exclude list is [], retry=0 2024-12-04T19:25:10,029 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:25:10,029 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:25:10,029 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:25:10,052 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340310014 2024-12-04T19:25:10,053 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:33421:33421)] 2024-12-04T19:25:10,053 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c53807fc839cea0d2df99fd1d43cdf76, NAME => 'testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76.', STARTKEY => '', ENDKEY => ''} 2024-12-04T19:25:10,053 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:25:10,053 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:10,053 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:10,054 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:10,055 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c53807fc839cea0d2df99fd1d43cdf76 columnFamilyName a 2024-12-04T19:25:10,055 DEBUG [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:10,062 DEBUG [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/a/9451179a3b604ab892aaacc2967e3ec3 2024-12-04T19:25:10,062 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(327): Store=c53807fc839cea0d2df99fd1d43cdf76/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:10,062 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:10,063 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c53807fc839cea0d2df99fd1d43cdf76 columnFamilyName b 2024-12-04T19:25:10,063 DEBUG [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:10,064 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(327): Store=c53807fc839cea0d2df99fd1d43cdf76/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:10,064 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:10,064 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c53807fc839cea0d2df99fd1d43cdf76 columnFamilyName c 2024-12-04T19:25:10,064 DEBUG [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:10,065 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(327): Store=c53807fc839cea0d2df99fd1d43cdf76/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:10,065 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:10,065 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:10,066 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:10,067 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000035 2024-12-04T19:25:10,089 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:25:10,090 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000035 2024-12-04T19:25:10,090 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c53807fc839cea0d2df99fd1d43cdf76 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-04T19:25:10,104 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/.tmp/b/04df9aa7180148dba32f1b14b3d442e3 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733340309919/Put/seqid=0 2024-12-04T19:25:10,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741936_1115 (size=5958) 2024-12-04T19:25:10,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741936_1115 (size=5958) 2024-12-04T19:25:10,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741936_1115 (size=5958) 2024-12-04T19:25:10,110 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/.tmp/b/04df9aa7180148dba32f1b14b3d442e3 2024-12-04T19:25:10,129 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/.tmp/c/355aa08d904b4782a2d301c84318f471 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733340309927/Put/seqid=0 2024-12-04T19:25:10,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741937_1116 (size=5958) 2024-12-04T19:25:10,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741937_1116 (size=5958) 2024-12-04T19:25:10,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741937_1116 (size=5958) 2024-12-04T19:25:10,135 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/.tmp/c/355aa08d904b4782a2d301c84318f471 2024-12-04T19:25:10,141 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/.tmp/b/04df9aa7180148dba32f1b14b3d442e3 as hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/b/04df9aa7180148dba32f1b14b3d442e3 2024-12-04T19:25:10,146 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/b/04df9aa7180148dba32f1b14b3d442e3, entries=10, sequenceid=35, filesize=5.8 K 2024-12-04T19:25:10,147 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/.tmp/c/355aa08d904b4782a2d301c84318f471 as hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/c/355aa08d904b4782a2d301c84318f471 2024-12-04T19:25:10,152 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/c/355aa08d904b4782a2d301c84318f471, entries=10, sequenceid=35, filesize=5.8 K 2024-12-04T19:25:10,153 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for c53807fc839cea0d2df99fd1d43cdf76 in 63ms, sequenceid=35, compaction requested=false; wal=null 2024-12-04T19:25:10,153 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000035 2024-12-04T19:25:10,154 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:10,154 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:10,155 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:25:10,156 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:10,158 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-04T19:25:10,159 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c53807fc839cea0d2df99fd1d43cdf76; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68811684, jitterRate=0.025373995304107666}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:25:10,159 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c53807fc839cea0d2df99fd1d43cdf76: Writing region info on filesystem at 1733340310053Initializing all the Stores at 1733340310054 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340310054Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340310054Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340310054Obtaining lock to block concurrent updates at 1733340310090 (+36 ms)Preparing flush snapshotting stores in c53807fc839cea0d2df99fd1d43cdf76 at 1733340310090Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733340310090Flushing stores of testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76. at 1733340310090Flushing c53807fc839cea0d2df99fd1d43cdf76/b: creating writer at 1733340310091 (+1 ms)Flushing c53807fc839cea0d2df99fd1d43cdf76/b: appending metadata at 1733340310104 (+13 ms)Flushing c53807fc839cea0d2df99fd1d43cdf76/b: closing flushed file at 1733340310104Flushing c53807fc839cea0d2df99fd1d43cdf76/c: creating writer at 1733340310114 (+10 ms)Flushing c53807fc839cea0d2df99fd1d43cdf76/c: appending metadata at 1733340310129 (+15 ms)Flushing c53807fc839cea0d2df99fd1d43cdf76/c: closing flushed file at 1733340310129Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c8b29c9: reopening flushed file at 1733340310140 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@135ddfac: reopening flushed file at 1733340310147 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for c53807fc839cea0d2df99fd1d43cdf76 in 63ms, sequenceid=35, compaction requested=false; wal=null at 1733340310153 (+6 ms)Cleaning up temporary data from old regions at 1733340310154 (+1 ms)Region opened successfully at 1733340310159 (+5 ms) 2024-12-04T19:25:10,239 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340310014, size=0 (0bytes) 2024-12-04T19:25:10,239 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340310014 might be still open, length is 0 2024-12-04T19:25:10,239 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340310014 2024-12-04T19:25:10,239 WARN [IPC Server handler 4 on default port 33439 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340310014 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-12-04T19:25:10,239 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340310014 after 0ms 2024-12-04T19:25:10,816 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:54856 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:41199:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54856 dst: /127.0.0.1:41199 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41199 remote=/127.0.0.1:54856]. Total timeout mills is 60000, 59370 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:25:10,817 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:43054 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:38107:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43054 dst: /127.0.0.1:38107 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:25:10,817 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-334028975_22 at /127.0.0.1:32938 [Receiving block BP-1762372386-172.17.0.2-1733340268509:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:44885:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:32938 dst: /127.0.0.1:44885 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:25:10,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741935_1117 (size=5567) 2024-12-04T19:25:10,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741935_1117 (size=5567) 2024-12-04T19:25:10,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741935_1117 (size=5567) 2024-12-04T19:25:12,980 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-04T19:25:12,980 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-04T19:25:12,983 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-04T19:25:12,983 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-04T19:25:14,241 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340310014 after 4002ms 2024-12-04T19:25:14,335 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:25:14,335 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340310014 took 4097ms 2024-12-04T19:25:14,337 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340310014; continuing. 2024-12-04T19:25:14,337 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340310014 so closing down 2024-12-04T19:25:14,337 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T19:25:14,338 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733340310014.temp 2024-12-04T19:25:14,361 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000037-wal.1733340310014.temp 2024-12-04T19:25:14,362 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T19:25:14,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741938_1118 (size=5575) 2024-12-04T19:25:14,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741938_1118 (size=5575) 2024-12-04T19:25:14,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741938_1118 (size=5575) 2024-12-04T19:25:14,368 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000037-wal.1733340310014.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-04T19:25:14,369 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000037-wal.1733340310014.temp to hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000066 2024-12-04T19:25:14,369 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340310014, size=0, length=0, corrupted=false, cancelled=false 2024-12-04T19:25:14,369 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340310014, journal: Splitting hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340310014, size=0 (0bytes) at 1733340310239Finishing writing output for hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340310014 so closing down at 1733340314337 (+4098 ms)Creating recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000037-wal.1733340310014.temp at 1733340314361 (+24 ms)3 split writer threads finished at 1733340314362 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000037-wal.1733340310014.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733340314368 (+6 ms)Rename recovered edits hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000037-wal.1733340310014.temp to hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000066 at 1733340314369 (+1 ms)Processed 30 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340310014, size=0, length=0, corrupted=false, cancelled=false at 1733340314369 2024-12-04T19:25:14,370 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340310014 to hdfs://localhost:33439/hbase/oldWALs/wal.1733340310014 2024-12-04T19:25:14,371 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000066 2024-12-04T19:25:14,371 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T19:25:14,373 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33439/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351, archiveDir=hdfs://localhost:33439/hbase/oldWALs, maxLogs=32 2024-12-04T19:25:14,385 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340314373, exclude list is [], retry=0 2024-12-04T19:25:14,387 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41199,DS-e3cbe799-2541-4ab3-b228-0635a0f35d5a,DISK] 2024-12-04T19:25:14,388 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44885,DS-e5ebbdb3-f383-40bf-b110-a17e202087e2,DISK] 2024-12-04T19:25:14,388 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38107,DS-2d6b14f1-9595-47ed-ac41-e8c8d9e08eac,DISK] 2024-12-04T19:25:14,410 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340314373 2024-12-04T19:25:14,410 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42335:42335),(127.0.0.1/127.0.0.1:37675:37675),(127.0.0.1/127.0.0.1:33421:33421)] 2024-12-04T19:25:14,410 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T19:25:14,412 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:14,412 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c53807fc839cea0d2df99fd1d43cdf76 columnFamilyName a 2024-12-04T19:25:14,412 DEBUG [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:14,417 DEBUG [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/a/9451179a3b604ab892aaacc2967e3ec3 2024-12-04T19:25:14,417 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(327): Store=c53807fc839cea0d2df99fd1d43cdf76/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:14,417 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:14,418 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c53807fc839cea0d2df99fd1d43cdf76 columnFamilyName b 2024-12-04T19:25:14,418 DEBUG [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:14,423 DEBUG [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/b/04df9aa7180148dba32f1b14b3d442e3 2024-12-04T19:25:14,423 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(327): Store=c53807fc839cea0d2df99fd1d43cdf76/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:14,423 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:14,424 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c53807fc839cea0d2df99fd1d43cdf76 columnFamilyName c 2024-12-04T19:25:14,424 DEBUG [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T19:25:14,428 DEBUG [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/c/355aa08d904b4782a2d301c84318f471 2024-12-04T19:25:14,428 INFO [StoreOpener-c53807fc839cea0d2df99fd1d43cdf76-1 {}] regionserver.HStore(327): Store=c53807fc839cea0d2df99fd1d43cdf76/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T19:25:14,428 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:14,429 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:14,430 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:14,431 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000066 2024-12-04T19:25:14,453 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T19:25:14,457 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000066 2024-12-04T19:25:14,457 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c53807fc839cea0d2df99fd1d43cdf76 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-04T19:25:14,469 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/.tmp/a/4c09da22035a438fac80cea547984990 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733340310166/Put/seqid=0 2024-12-04T19:25:14,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741940_1120 (size=5958) 2024-12-04T19:25:14,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741940_1120 (size=5958) 2024-12-04T19:25:14,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741940_1120 (size=5958) 2024-12-04T19:25:14,477 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/.tmp/a/4c09da22035a438fac80cea547984990 2024-12-04T19:25:14,493 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/.tmp/b/f5ab3dd17e8741539d91f8030c700bb1 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733340310174/Put/seqid=0 2024-12-04T19:25:14,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741941_1121 (size=5958) 2024-12-04T19:25:14,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741941_1121 (size=5958) 2024-12-04T19:25:14,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741941_1121 (size=5958) 2024-12-04T19:25:14,499 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/.tmp/b/f5ab3dd17e8741539d91f8030c700bb1 2024-12-04T19:25:14,515 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/.tmp/c/f3ccd9f959a546eda564fe43ad919fa4 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733340310180/Put/seqid=0 2024-12-04T19:25:14,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741942_1122 (size=5958) 2024-12-04T19:25:14,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741942_1122 (size=5958) 2024-12-04T19:25:14,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741942_1122 (size=5958) 2024-12-04T19:25:14,521 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/.tmp/c/f3ccd9f959a546eda564fe43ad919fa4 2024-12-04T19:25:14,526 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/.tmp/a/4c09da22035a438fac80cea547984990 as hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/a/4c09da22035a438fac80cea547984990 2024-12-04T19:25:14,530 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/a/4c09da22035a438fac80cea547984990, entries=10, sequenceid=66, filesize=5.8 K 2024-12-04T19:25:14,531 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/.tmp/b/f5ab3dd17e8741539d91f8030c700bb1 as hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/b/f5ab3dd17e8741539d91f8030c700bb1 2024-12-04T19:25:14,536 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/b/f5ab3dd17e8741539d91f8030c700bb1, entries=10, sequenceid=66, filesize=5.8 K 2024-12-04T19:25:14,537 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/.tmp/c/f3ccd9f959a546eda564fe43ad919fa4 as hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/c/f3ccd9f959a546eda564fe43ad919fa4 2024-12-04T19:25:14,541 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/c/f3ccd9f959a546eda564fe43ad919fa4, entries=10, sequenceid=66, filesize=5.8 K 2024-12-04T19:25:14,541 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for c53807fc839cea0d2df99fd1d43cdf76 in 84ms, sequenceid=66, compaction requested=false; wal=null 2024-12-04T19:25:14,542 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/0000000000000000066 2024-12-04T19:25:14,543 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:14,543 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:14,543 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T19:25:14,544 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c53807fc839cea0d2df99fd1d43cdf76 2024-12-04T19:25:14,546 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/hbase/data/default/testReplayEditsWrittenViaHRegion/c53807fc839cea0d2df99fd1d43cdf76/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-04T19:25:14,547 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c53807fc839cea0d2df99fd1d43cdf76; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72198234, jitterRate=0.07583752274513245}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T19:25:14,547 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c53807fc839cea0d2df99fd1d43cdf76: Writing region info on filesystem at 1733340314410Initializing all the Stores at 1733340314411 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340314411Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340314411Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733340314411Obtaining lock to block concurrent updates at 1733340314457 (+46 ms)Preparing flush snapshotting stores in c53807fc839cea0d2df99fd1d43cdf76 at 1733340314457Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733340314457Flushing stores of testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76. at 1733340314457Flushing c53807fc839cea0d2df99fd1d43cdf76/a: creating writer at 1733340314457Flushing c53807fc839cea0d2df99fd1d43cdf76/a: appending metadata at 1733340314469 (+12 ms)Flushing c53807fc839cea0d2df99fd1d43cdf76/a: closing flushed file at 1733340314469Flushing c53807fc839cea0d2df99fd1d43cdf76/b: creating writer at 1733340314481 (+12 ms)Flushing c53807fc839cea0d2df99fd1d43cdf76/b: appending metadata at 1733340314492 (+11 ms)Flushing c53807fc839cea0d2df99fd1d43cdf76/b: closing flushed file at 1733340314492Flushing c53807fc839cea0d2df99fd1d43cdf76/c: creating writer at 1733340314503 (+11 ms)Flushing c53807fc839cea0d2df99fd1d43cdf76/c: appending metadata at 1733340314514 (+11 ms)Flushing c53807fc839cea0d2df99fd1d43cdf76/c: closing flushed file at 1733340314514Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f517cd0: reopening flushed file at 1733340314525 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55feb9b8: reopening flushed file at 1733340314530 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62911f50: reopening flushed file at 1733340314536 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for c53807fc839cea0d2df99fd1d43cdf76 in 84ms, sequenceid=66, compaction requested=false; wal=null at 1733340314541 (+5 ms)Cleaning up temporary data from old regions at 1733340314543 (+2 ms)Region opened successfully at 1733340314547 (+4 ms) 2024-12-04T19:25:14,557 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c53807fc839cea0d2df99fd1d43cdf76, disabling compactions & flushes 2024-12-04T19:25:14,557 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76. 2024-12-04T19:25:14,557 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76. 2024-12-04T19:25:14,557 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76. after waiting 0 ms 2024-12-04T19:25:14,557 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76. 2024-12-04T19:25:14,558 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733340309352.c53807fc839cea0d2df99fd1d43cdf76. 2024-12-04T19:25:14,558 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c53807fc839cea0d2df99fd1d43cdf76: Waiting for close lock at 1733340314557Disabling compacts and flushes for region at 1733340314557Disabling writes for close at 1733340314557Writing region close event to WAL at 1733340314558 (+1 ms)Closed at 1733340314558 2024-12-04T19:25:14,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741939_1119 (size=175) 2024-12-04T19:25:14,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741939_1119 (size=175) 2024-12-04T19:25:14,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741939_1119 (size=175) 2024-12-04T19:25:14,562 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T19:25:14,562 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733340314373) 2024-12-04T19:25:14,574 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=444 (was 438) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:33439 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1199048880_22 at /127.0.0.1:36442 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:33439 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1199048880_22 at /127.0.0.1:49930 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1199048880_22 at /127.0.0.1:58524 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1410 (was 1343) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=190 (was 181) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=15500 (was 15509) 2024-12-04T19:25:14,574 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1410 is superior to 1024 2024-12-04T19:25:14,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T19:25:14,574 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T19:25:14,574 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T19:25:14,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:25:14,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:25:14,575 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T19:25:14,575 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T19:25:14,575 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=526865473, stopped=false 2024-12-04T19:25:14,575 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9ba2734cd39f,44405,1733340272667 2024-12-04T19:25:14,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T19:25:14,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T19:25:14,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T19:25:14,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:25:14,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:25:14,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:25:14,615 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T19:25:14,616 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T19:25:14,616 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T19:25:14,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:25:14,616 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9ba2734cd39f,42371,1733340273381' ***** 2024-12-04T19:25:14,616 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T19:25:14,616 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T19:25:14,616 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9ba2734cd39f,38513,1733340273533' ***** 2024-12-04T19:25:14,616 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T19:25:14,616 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T19:25:14,616 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T19:25:14,616 INFO [RS:0;9ba2734cd39f:42371 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T19:25:14,616 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T19:25:14,617 INFO [RS:2;9ba2734cd39f:38513 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T19:25:14,617 INFO [RS:0;9ba2734cd39f:42371 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T19:25:14,617 INFO [RS:2;9ba2734cd39f:38513 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T19:25:14,617 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T19:25:14,617 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.HRegionServer(3091): Received CLOSE for 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:25:14,617 INFO [RS:2;9ba2734cd39f:38513 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T19:25:14,617 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.HRegionServer(959): stopping server 9ba2734cd39f,42371,1733340273381 2024-12-04T19:25:14,617 INFO [RS:2;9ba2734cd39f:38513 {}] regionserver.HRegionServer(959): stopping server 9ba2734cd39f,38513,1733340273533 2024-12-04T19:25:14,617 INFO [RS:2;9ba2734cd39f:38513 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T19:25:14,617 INFO [RS:0;9ba2734cd39f:42371 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T19:25:14,617 INFO [RS:0;9ba2734cd39f:42371 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9ba2734cd39f:42371. 2024-12-04T19:25:14,617 INFO [RS:2;9ba2734cd39f:38513 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;9ba2734cd39f:38513. 2024-12-04T19:25:14,617 DEBUG [RS:0;9ba2734cd39f:42371 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T19:25:14,617 DEBUG [RS:2;9ba2734cd39f:38513 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T19:25:14,617 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 88a3d8cc28447f8692a6ef68159edd5f, disabling compactions & flushes 2024-12-04T19:25:14,617 DEBUG [RS:0;9ba2734cd39f:42371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:25:14,617 DEBUG [RS:2;9ba2734cd39f:38513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:25:14,617 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:25:14,617 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T19:25:14,617 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:25:14,617 INFO [RS:2;9ba2734cd39f:38513 {}] regionserver.HRegionServer(976): stopping server 9ba2734cd39f,38513,1733340273533; all regions closed. 2024-12-04T19:25:14,617 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T19:25:14,617 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T19:25:14,617 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. after waiting 0 ms 2024-12-04T19:25:14,617 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:25:14,617 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T19:25:14,618 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T19:25:14,618 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-04T19:25:14,618 DEBUG [RS:0;9ba2734cd39f:42371 {}] regionserver.HRegionServer(1325): Online Regions={88a3d8cc28447f8692a6ef68159edd5f=testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f., 1588230740=hbase:meta,,1.1588230740} 2024-12-04T19:25:14,618 DEBUG [RS:0;9ba2734cd39f:42371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 88a3d8cc28447f8692a6ef68159edd5f 2024-12-04T19:25:14,618 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T19:25:14,618 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T19:25:14,618 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T19:25:14,618 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T19:25:14,618 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T19:25:14,618 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-12-04T19:25:14,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741834_1010 (size=175) 2024-12-04T19:25:14,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741834_1010 (size=175) 2024-12-04T19:25:14,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741834_1010 (size=175) 2024-12-04T19:25:14,623 DEBUG [RS:2;9ba2734cd39f:38513 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/oldWALs 2024-12-04T19:25:14,623 INFO [RS:2;9ba2734cd39f:38513 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9ba2734cd39f%2C38513%2C1733340273533:(num 1733340275242) 2024-12-04T19:25:14,623 DEBUG [RS:2;9ba2734cd39f:38513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:25:14,624 INFO [RS:2;9ba2734cd39f:38513 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T19:25:14,624 INFO [RS:2;9ba2734cd39f:38513 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T19:25:14,624 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/default/testReplayEditsAfterRegionMovedWithMultiCF/88a3d8cc28447f8692a6ef68159edd5f/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-04T19:25:14,624 INFO [RS:2;9ba2734cd39f:38513 {}] hbase.ChoreService(370): Chore service for: regionserver/9ba2734cd39f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T19:25:14,624 INFO [RS:2;9ba2734cd39f:38513 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T19:25:14,624 INFO [RS:2;9ba2734cd39f:38513 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T19:25:14,624 INFO [regionserver/9ba2734cd39f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T19:25:14,624 INFO [RS:2;9ba2734cd39f:38513 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T19:25:14,624 INFO [RS:2;9ba2734cd39f:38513 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T19:25:14,624 INFO [RS:2;9ba2734cd39f:38513 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38513 2024-12-04T19:25:14,625 INFO [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:25:14,625 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 88a3d8cc28447f8692a6ef68159edd5f: Waiting for close lock at 1733340314617Running coprocessor pre-close hooks at 1733340314617Disabling compacts and flushes for region at 1733340314617Disabling writes for close at 1733340314617Writing region close event to WAL at 1733340314619 (+2 ms)Running coprocessor post-close hooks at 1733340314625 (+6 ms)Closed at 1733340314625 2024-12-04T19:25:14,625 DEBUG [RS_CLOSE_REGION-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f. 2024-12-04T19:25:14,625 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/.tmp/info/06f24794bb8f4b439c2f518748496e8f is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733340288923.88a3d8cc28447f8692a6ef68159edd5f./info:regioninfo/1733340292988/Put/seqid=0 2024-12-04T19:25:14,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741943_1123 (size=6778) 2024-12-04T19:25:14,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741943_1123 (size=6778) 2024-12-04T19:25:14,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741943_1123 (size=6778) 2024-12-04T19:25:14,635 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/.tmp/info/06f24794bb8f4b439c2f518748496e8f 2024-12-04T19:25:14,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9ba2734cd39f,38513,1733340273533 2024-12-04T19:25:14,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T19:25:14,636 INFO [RS:2;9ba2734cd39f:38513 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T19:25:14,637 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9ba2734cd39f,38513,1733340273533] 2024-12-04T19:25:14,641 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/.tmp/info/06f24794bb8f4b439c2f518748496e8f as hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/info/06f24794bb8f4b439c2f518748496e8f 2024-12-04T19:25:14,643 INFO [regionserver/9ba2734cd39f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T19:25:14,643 INFO [regionserver/9ba2734cd39f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T19:25:14,646 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/info/06f24794bb8f4b439c2f518748496e8f, entries=8, sequenceid=23, filesize=6.6 K 2024-12-04T19:25:14,647 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 29ms, sequenceid=23, compaction requested=false 2024-12-04T19:25:14,651 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-12-04T19:25:14,652 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T19:25:14,652 INFO [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T19:25:14,652 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733340314618Running coprocessor pre-close hooks at 1733340314618Disabling compacts and flushes for region at 1733340314618Disabling writes for close at 1733340314618Obtaining lock to block concurrent updates at 1733340314618Preparing flush snapshotting stores in 1588230740 at 1733340314618Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1733340314619 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733340314621 (+2 ms)Flushing 1588230740/info: creating writer at 1733340314621Flushing 1588230740/info: appending metadata at 1733340314625 (+4 ms)Flushing 1588230740/info: closing flushed file at 1733340314625Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74c8a305: reopening flushed file at 1733340314640 (+15 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 29ms, sequenceid=23, compaction requested=false at 1733340314647 (+7 ms)Writing region close event to WAL at 1733340314648 (+1 ms)Running coprocessor post-close hooks at 1733340314652 (+4 ms)Closed at 1733340314652 2024-12-04T19:25:14,652 DEBUG [RS_CLOSE_META-regionserver/9ba2734cd39f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T19:25:14,731 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9ba2734cd39f,38513,1733340273533 already deleted, retry=false 2024-12-04T19:25:14,731 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9ba2734cd39f,38513,1733340273533 expired; onlineServers=1 2024-12-04T19:25:14,818 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.HRegionServer(976): stopping server 9ba2734cd39f,42371,1733340273381; all regions closed. 2024-12-04T19:25:14,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T19:25:14,821 INFO [RS:2;9ba2734cd39f:38513 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T19:25:14,821 INFO [RS:2;9ba2734cd39f:38513 {}] regionserver.HRegionServer(1031): Exiting; stopping=9ba2734cd39f,38513,1733340273533; zookeeper connection closed. 2024-12-04T19:25:14,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38513-0x101a1cd90bd0003, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T19:25:14,822 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@47999aa9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@47999aa9 2024-12-04T19:25:14,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741893_1071 (size=2625) 2024-12-04T19:25:14,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741893_1071 (size=2625) 2024-12-04T19:25:14,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741893_1071 (size=2625) 2024-12-04T19:25:14,829 DEBUG [RS:0;9ba2734cd39f:42371 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/oldWALs 2024-12-04T19:25:14,829 INFO [RS:0;9ba2734cd39f:42371 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9ba2734cd39f%2C42371%2C1733340273381.meta:.meta(num 1733340292065) 2024-12-04T19:25:14,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741835_1011 (size=3621) 2024-12-04T19:25:14,832 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/WALs/9ba2734cd39f,42371,1733340273381/9ba2734cd39f%2C42371%2C1733340273381.1733340275242 not finished, retry = 0 2024-12-04T19:25:14,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741835_1011 (size=3621) 2024-12-04T19:25:14,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741835_1011 (size=3621) 2024-12-04T19:25:14,937 DEBUG [RS:0;9ba2734cd39f:42371 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/oldWALs 2024-12-04T19:25:14,938 INFO [RS:0;9ba2734cd39f:42371 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9ba2734cd39f%2C42371%2C1733340273381:(num 1733340275242) 2024-12-04T19:25:14,938 DEBUG [RS:0;9ba2734cd39f:42371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T19:25:14,938 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T19:25:14,938 INFO [RS:0;9ba2734cd39f:42371 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T19:25:14,939 INFO [RS:0;9ba2734cd39f:42371 {}] hbase.ChoreService(370): Chore service for: regionserver/9ba2734cd39f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T19:25:14,939 INFO [RS:0;9ba2734cd39f:42371 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T19:25:14,939 INFO [regionserver/9ba2734cd39f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T19:25:14,940 INFO [RS:0;9ba2734cd39f:42371 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42371 2024-12-04T19:25:14,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9ba2734cd39f,42371,1733340273381 2024-12-04T19:25:14,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T19:25:14,952 INFO [RS:0;9ba2734cd39f:42371 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T19:25:14,962 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9ba2734cd39f,42371,1733340273381] 2024-12-04T19:25:14,973 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9ba2734cd39f,42371,1733340273381 already deleted, retry=false 2024-12-04T19:25:14,973 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9ba2734cd39f,42371,1733340273381 expired; onlineServers=0 2024-12-04T19:25:14,973 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9ba2734cd39f,44405,1733340272667' ***** 2024-12-04T19:25:14,973 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T19:25:14,973 INFO [M:0;9ba2734cd39f:44405 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T19:25:14,974 INFO [M:0;9ba2734cd39f:44405 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T19:25:14,974 DEBUG [M:0;9ba2734cd39f:44405 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T19:25:14,974 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T19:25:14,974 DEBUG [M:0;9ba2734cd39f:44405 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T19:25:14,974 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster-HFileCleaner.small.0-1733340274829 {}] cleaner.HFileCleaner(306): Exit Thread[master/9ba2734cd39f:0:becomeActiveMaster-HFileCleaner.small.0-1733340274829,5,FailOnTimeoutGroup] 2024-12-04T19:25:14,974 DEBUG [master/9ba2734cd39f:0:becomeActiveMaster-HFileCleaner.large.0-1733340274828 {}] cleaner.HFileCleaner(306): Exit Thread[master/9ba2734cd39f:0:becomeActiveMaster-HFileCleaner.large.0-1733340274828,5,FailOnTimeoutGroup] 2024-12-04T19:25:14,975 INFO [M:0;9ba2734cd39f:44405 {}] hbase.ChoreService(370): Chore service for: master/9ba2734cd39f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T19:25:14,975 INFO [M:0;9ba2734cd39f:44405 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T19:25:14,975 DEBUG [M:0;9ba2734cd39f:44405 {}] master.HMaster(1795): Stopping service threads 2024-12-04T19:25:14,975 INFO [M:0;9ba2734cd39f:44405 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T19:25:14,975 INFO [M:0;9ba2734cd39f:44405 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T19:25:14,976 INFO [M:0;9ba2734cd39f:44405 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T19:25:14,977 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T19:25:14,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T19:25:14,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T19:25:14,984 DEBUG [M:0;9ba2734cd39f:44405 {}] zookeeper.ZKUtil(347): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T19:25:14,984 WARN [M:0;9ba2734cd39f:44405 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T19:25:14,985 INFO [M:0;9ba2734cd39f:44405 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/.lastflushedseqids 2024-12-04T19:25:14,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741944_1124 (size=119) 2024-12-04T19:25:14,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741944_1124 (size=119) 2024-12-04T19:25:14,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741944_1124 (size=119) 2024-12-04T19:25:15,000 INFO [M:0;9ba2734cd39f:44405 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T19:25:15,000 INFO [M:0;9ba2734cd39f:44405 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T19:25:15,000 DEBUG [M:0;9ba2734cd39f:44405 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T19:25:15,000 INFO [M:0;9ba2734cd39f:44405 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:25:15,000 DEBUG [M:0;9ba2734cd39f:44405 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:25:15,000 DEBUG [M:0;9ba2734cd39f:44405 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T19:25:15,000 DEBUG [M:0;9ba2734cd39f:44405 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:25:15,000 INFO [M:0;9ba2734cd39f:44405 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.38 KB heapSize=102.70 KB 2024-12-04T19:25:15,016 DEBUG [M:0;9ba2734cd39f:44405 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d5b2da950dc6471e9c6c04952b73698e is 82, key is hbase:meta,,1/info:regioninfo/1733340292256/Put/seqid=0 2024-12-04T19:25:15,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741945_1125 (size=6063) 2024-12-04T19:25:15,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741945_1125 (size=6063) 2024-12-04T19:25:15,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741945_1125 (size=6063) 2024-12-04T19:25:15,021 INFO [M:0;9ba2734cd39f:44405 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d5b2da950dc6471e9c6c04952b73698e 2024-12-04T19:25:15,039 DEBUG [M:0;9ba2734cd39f:44405 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b97e68f3ac714b7aa2968cfa26ab64d5 is 1075, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733340289359/Put/seqid=0 2024-12-04T19:25:15,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741946_1126 (size=7906) 2024-12-04T19:25:15,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741946_1126 (size=7906) 2024-12-04T19:25:15,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741946_1126 (size=7906) 2024-12-04T19:25:15,045 INFO [M:0;9ba2734cd39f:44405 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.16 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b97e68f3ac714b7aa2968cfa26ab64d5 2024-12-04T19:25:15,049 INFO [M:0;9ba2734cd39f:44405 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b97e68f3ac714b7aa2968cfa26ab64d5 2024-12-04T19:25:15,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T19:25:15,063 INFO [RS:0;9ba2734cd39f:42371 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T19:25:15,063 INFO [RS:0;9ba2734cd39f:42371 {}] regionserver.HRegionServer(1031): Exiting; stopping=9ba2734cd39f,42371,1733340273381; zookeeper connection closed. 2024-12-04T19:25:15,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42371-0x101a1cd90bd0001, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T19:25:15,063 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@15a81787 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@15a81787 2024-12-04T19:25:15,063 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-04T19:25:15,065 DEBUG [M:0;9ba2734cd39f:44405 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8345d005d76247beb92832c6b4ab2806 is 69, key is 9ba2734cd39f,38513,1733340273533/rs:state/1733340274957/Put/seqid=0 2024-12-04T19:25:15,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741947_1127 (size=5440) 2024-12-04T19:25:15,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741947_1127 (size=5440) 2024-12-04T19:25:15,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741947_1127 (size=5440) 2024-12-04T19:25:15,071 INFO [M:0;9ba2734cd39f:44405 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8345d005d76247beb92832c6b4ab2806 2024-12-04T19:25:15,075 INFO [M:0;9ba2734cd39f:44405 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8345d005d76247beb92832c6b4ab2806 2024-12-04T19:25:15,076 DEBUG [M:0;9ba2734cd39f:44405 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d5b2da950dc6471e9c6c04952b73698e as hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d5b2da950dc6471e9c6c04952b73698e 2024-12-04T19:25:15,080 INFO [M:0;9ba2734cd39f:44405 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d5b2da950dc6471e9c6c04952b73698e, entries=14, sequenceid=207, filesize=5.9 K 2024-12-04T19:25:15,081 DEBUG [M:0;9ba2734cd39f:44405 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b97e68f3ac714b7aa2968cfa26ab64d5 as hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b97e68f3ac714b7aa2968cfa26ab64d5 2024-12-04T19:25:15,085 INFO [M:0;9ba2734cd39f:44405 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b97e68f3ac714b7aa2968cfa26ab64d5 2024-12-04T19:25:15,085 INFO [M:0;9ba2734cd39f:44405 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b97e68f3ac714b7aa2968cfa26ab64d5, entries=21, sequenceid=207, filesize=7.7 K 2024-12-04T19:25:15,086 DEBUG [M:0;9ba2734cd39f:44405 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8345d005d76247beb92832c6b4ab2806 as hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8345d005d76247beb92832c6b4ab2806 2024-12-04T19:25:15,089 INFO [M:0;9ba2734cd39f:44405 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8345d005d76247beb92832c6b4ab2806 2024-12-04T19:25:15,090 INFO [M:0;9ba2734cd39f:44405 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33439/user/jenkins/test-data/4a184511-9752-e116-45ae-264985591890/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8345d005d76247beb92832c6b4ab2806, entries=3, sequenceid=207, filesize=5.3 K 2024-12-04T19:25:15,090 INFO [M:0;9ba2734cd39f:44405 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 90ms, sequenceid=207, compaction requested=false 2024-12-04T19:25:15,092 INFO [M:0;9ba2734cd39f:44405 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T19:25:15,092 DEBUG [M:0;9ba2734cd39f:44405 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733340315000Disabling compacts and flushes for region at 1733340315000Disabling writes for close at 1733340315000Obtaining lock to block concurrent updates at 1733340315000Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733340315000Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85386, getHeapSize=105096, getOffHeapSize=0, getCellsCount=248 at 1733340315001 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733340315001Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733340315001Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733340315015 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733340315015Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733340315025 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733340315038 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733340315038Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733340315050 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733340315064 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733340315064Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f79d3c0: reopening flushed file at 1733340315075 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2635f97e: reopening flushed file at 1733340315080 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e095c85: reopening flushed file at 1733340315085 (+5 ms)Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 90ms, sequenceid=207, compaction requested=false at 1733340315091 (+6 ms)Writing region close event to WAL at 1733340315092 (+1 ms)Closed at 1733340315092 2024-12-04T19:25:15,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41199 is added to blk_1073741830_1006 (size=102150) 2024-12-04T19:25:15,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38107 is added to blk_1073741830_1006 (size=102150) 2024-12-04T19:25:15,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44885 is added to blk_1073741830_1006 (size=102150) 2024-12-04T19:25:15,095 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T19:25:15,095 INFO [M:0;9ba2734cd39f:44405 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T19:25:15,095 INFO [M:0;9ba2734cd39f:44405 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44405 2024-12-04T19:25:15,095 INFO [M:0;9ba2734cd39f:44405 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T19:25:15,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T19:25:15,205 INFO [M:0;9ba2734cd39f:44405 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T19:25:15,205 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44405-0x101a1cd90bd0000, quorum=127.0.0.1:61732, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T19:25:15,217 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340294117/wal.1733340303854 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340294117/wal.1733340303854 (inode 16688) Holder DFSClient_NONMAPREDUCE_-334028975_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733340294117/wal.1733340303854 (inode 16688) Holder DFSClient_NONMAPREDUCE_-334028975_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-04T19:25:15,220 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521/wal.1733340293952 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521/wal.1733340293952 (inode 16665) Holder DFSClient_NONMAPREDUCE_-334028975_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733340293521/wal.1733340293952 (inode 16665) Holder DFSClient_NONMAPREDUCE_-334028975_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-04T19:25:15,221 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733340284442/wal.1733340284555 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:25:15,222 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733340304094/wal.1733340304656 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733340304094/wal.1733340304656 (inode 16714) Holder DFSClient_NONMAPREDUCE_-334028975_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733340304094/wal.1733340304656 (inode 16714) Holder DFSClient_NONMAPREDUCE_-334028975_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-04T19:25:15,223 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733340304843/wal.1733340304925 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:25:15,224 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733340309351/wal.1733340310014 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:25:15,224 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733340277223/wal.1733340277313 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T19:25:15,226 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal.1733340277061 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal.1733340277061 (inode 16485) Holder DFSClient_NONMAPREDUCE_-334028975_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733340276719/wal.1733340277061 (inode 16485) Holder DFSClient_NONMAPREDUCE_-334028975_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-04T19:25:15,229 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal.1733340276461 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal.1733340276461 (inode 16462) Holder DFSClient_NONMAPREDUCE_-334028975_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733340276083/wal.1733340276461 (inode 16462) Holder DFSClient_NONMAPREDUCE_-334028975_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-04T19:25:15,264 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@420bc738{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T19:25:15,267 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T19:25:15,267 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T19:25:15,267 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T19:25:15,267 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/hadoop.log.dir/,STOPPED} 2024-12-04T19:25:15,271 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T19:25:15,271 WARN [BP-1762372386-172.17.0.2-1733340268509 heartbeating to localhost/127.0.0.1:33439 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T19:25:15,271 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T19:25:15,271 WARN [BP-1762372386-172.17.0.2-1733340268509 heartbeating to localhost/127.0.0.1:33439 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1762372386-172.17.0.2-1733340268509 (Datanode Uuid 5808b0f9-ae19-459e-9dd8-c71922061aec) service to localhost/127.0.0.1:33439 2024-12-04T19:25:15,274 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/data/data5/current/BP-1762372386-172.17.0.2-1733340268509 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T19:25:15,274 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/data/data6/current/BP-1762372386-172.17.0.2-1733340268509 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T19:25:15,275 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T19:25:15,277 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64052af6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T19:25:15,277 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T19:25:15,277 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T19:25:15,277 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T19:25:15,277 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/hadoop.log.dir/,STOPPED} 2024-12-04T19:25:15,278 WARN [BP-1762372386-172.17.0.2-1733340268509 heartbeating to localhost/127.0.0.1:33439 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T19:25:15,279 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T19:25:15,279 WARN [BP-1762372386-172.17.0.2-1733340268509 heartbeating to localhost/127.0.0.1:33439 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1762372386-172.17.0.2-1733340268509 (Datanode Uuid 1e307788-1f8e-48c0-8bf9-4e0292381e9c) service to localhost/127.0.0.1:33439 2024-12-04T19:25:15,279 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T19:25:15,279 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/data/data3/current/BP-1762372386-172.17.0.2-1733340268509 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T19:25:15,279 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/data/data4/current/BP-1762372386-172.17.0.2-1733340268509 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T19:25:15,280 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T19:25:15,282 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@501095cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T19:25:15,282 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T19:25:15,282 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T19:25:15,282 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T19:25:15,283 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/hadoop.log.dir/,STOPPED} 2024-12-04T19:25:15,283 WARN [BP-1762372386-172.17.0.2-1733340268509 heartbeating to localhost/127.0.0.1:33439 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T19:25:15,283 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T19:25:15,283 WARN [BP-1762372386-172.17.0.2-1733340268509 heartbeating to localhost/127.0.0.1:33439 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1762372386-172.17.0.2-1733340268509 (Datanode Uuid b04cbcf5-11a1-42aa-a434-eab201f64137) service to localhost/127.0.0.1:33439 2024-12-04T19:25:15,283 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T19:25:15,284 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/data/data1/current/BP-1762372386-172.17.0.2-1733340268509 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T19:25:15,284 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/cluster_4d1a1e38-8117-6b39-c51f-3ff8b5f40805/data/data2/current/BP-1762372386-172.17.0.2-1733340268509 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T19:25:15,285 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T19:25:15,290 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T19:25:15,291 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T19:25:15,291 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T19:25:15,291 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T19:25:15,291 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/547cee7f-4b48-41cf-f806-1c4c3ab5dec1/hadoop.log.dir/,STOPPED} 2024-12-04T19:25:15,299 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T19:25:15,345 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down