2024-12-03 20:34:07,969 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-03 20:34:07,979 main DEBUG Took 0.008421 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 20:34:07,980 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 20:34:07,980 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 20:34:07,981 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 20:34:07,982 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 20:34:07,989 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 20:34:07,999 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 20:34:08,000 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 20:34:08,001 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 20:34:08,001 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 20:34:08,002 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 20:34:08,002 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 20:34:08,002 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 20:34:08,003 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 20:34:08,003 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 20:34:08,003 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 20:34:08,004 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 20:34:08,004 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 20:34:08,004 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 20:34:08,005 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 20:34:08,005 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 20:34:08,005 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 20:34:08,006 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 20:34:08,006 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 20:34:08,006 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 20:34:08,006 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 20:34:08,007 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 20:34:08,007 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 20:34:08,007 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 20:34:08,008 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 20:34:08,008 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 20:34:08,008 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 20:34:08,010 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 20:34:08,011 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 20:34:08,012 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 20:34:08,013 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 20:34:08,014 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 20:34:08,014 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 20:34:08,022 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 20:34:08,024 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 20:34:08,025 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 20:34:08,026 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 20:34:08,026 main DEBUG createAppenders(={Console}) 2024-12-03 20:34:08,027 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-03 20:34:08,027 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-03 20:34:08,027 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-03 20:34:08,027 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 20:34:08,028 main DEBUG OutputStream closed 2024-12-03 20:34:08,028 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 20:34:08,028 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 20:34:08,028 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-03 20:34:08,088 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 20:34:08,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 20:34:08,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 20:34:08,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 20:34:08,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 20:34:08,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 20:34:08,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 20:34:08,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 20:34:08,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 20:34:08,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 20:34:08,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 20:34:08,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 20:34:08,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 20:34:08,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 20:34:08,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 20:34:08,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 20:34:08,095 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 20:34:08,095 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 20:34:08,097 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 20:34:08,098 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-03 20:34:08,098 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 20:34:08,099 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-03T20:34:08,318 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7 2024-12-03 20:34:08,320 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 20:34:08,321 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T20:34:08,328 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-03T20:34:08,333 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayValueCompression timeout: 13 mins 2024-12-03T20:34:08,359 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T20:34:08,414 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T20:34:08,414 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T20:34:08,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T20:34:08,445 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1, deleteOnExit=true 2024-12-03T20:34:08,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T20:34:08,446 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/test.cache.data in system properties and HBase conf 2024-12-03T20:34:08,446 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T20:34:08,447 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/hadoop.log.dir in system properties and HBase conf 2024-12-03T20:34:08,447 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T20:34:08,447 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T20:34:08,448 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T20:34:08,515 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T20:34:08,598 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T20:34:08,601 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T20:34:08,602 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T20:34:08,602 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T20:34:08,603 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T20:34:08,603 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T20:34:08,603 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T20:34:08,604 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T20:34:08,604 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T20:34:08,605 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T20:34:08,605 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/nfs.dump.dir in system properties and HBase conf 2024-12-03T20:34:08,605 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/java.io.tmpdir in system properties and HBase conf 2024-12-03T20:34:08,606 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T20:34:08,606 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T20:34:08,606 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T20:34:09,649 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T20:34:09,713 INFO [Time-limited test {}] log.Log(170): Logging initialized @2395ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T20:34:09,777 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T20:34:09,831 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T20:34:09,849 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T20:34:09,849 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T20:34:09,851 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T20:34:09,874 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T20:34:09,878 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/hadoop.log.dir/,AVAILABLE} 2024-12-03T20:34:09,880 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T20:34:10,027 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3717288f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/java.io.tmpdir/jetty-localhost-42247-hadoop-hdfs-3_4_1-tests_jar-_-any-5794308521980492166/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T20:34:10,032 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:42247} 2024-12-03T20:34:10,033 INFO [Time-limited test {}] server.Server(415): Started @2715ms 2024-12-03T20:34:10,602 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T20:34:10,609 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T20:34:10,610 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T20:34:10,611 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T20:34:10,611 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T20:34:10,611 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/hadoop.log.dir/,AVAILABLE} 2024-12-03T20:34:10,612 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T20:34:10,720 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@330740de{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/java.io.tmpdir/jetty-localhost-38819-hadoop-hdfs-3_4_1-tests_jar-_-any-17342506368244797335/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T20:34:10,721 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:38819} 2024-12-03T20:34:10,721 INFO [Time-limited test {}] server.Server(415): Started @3404ms 2024-12-03T20:34:10,771 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T20:34:10,882 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T20:34:10,889 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T20:34:10,890 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T20:34:10,890 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T20:34:10,890 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T20:34:10,892 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/hadoop.log.dir/,AVAILABLE} 2024-12-03T20:34:10,893 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T20:34:11,004 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bd427b8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/java.io.tmpdir/jetty-localhost-45587-hadoop-hdfs-3_4_1-tests_jar-_-any-592515915764823867/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T20:34:11,005 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:45587} 2024-12-03T20:34:11,005 INFO [Time-limited test {}] server.Server(415): Started @3688ms 2024-12-03T20:34:11,008 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T20:34:11,040 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T20:34:11,044 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T20:34:11,045 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T20:34:11,046 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T20:34:11,046 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T20:34:11,047 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/hadoop.log.dir/,AVAILABLE} 2024-12-03T20:34:11,047 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T20:34:11,139 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@35f1150e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/java.io.tmpdir/jetty-localhost-46847-hadoop-hdfs-3_4_1-tests_jar-_-any-15113292877456841862/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T20:34:11,140 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:46847} 2024-12-03T20:34:11,140 INFO [Time-limited test {}] server.Server(415): Started @3823ms 2024-12-03T20:34:11,142 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T20:34:12,491 WARN [Thread-129 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/data/data2/current/BP-1199604065-172.17.0.3-1733258049083/current, will proceed with Du for space computation calculation, 2024-12-03T20:34:12,491 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/data/data5/current/BP-1199604065-172.17.0.3-1733258049083/current, will proceed with Du for space computation calculation, 2024-12-03T20:34:12,491 WARN [Thread-126 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/data/data3/current/BP-1199604065-172.17.0.3-1733258049083/current, will proceed with Du for space computation calculation, 2024-12-03T20:34:12,491 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/data/data6/current/BP-1199604065-172.17.0.3-1733258049083/current, will proceed with Du for space computation calculation, 2024-12-03T20:34:12,491 WARN [Thread-128 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/data/data1/current/BP-1199604065-172.17.0.3-1733258049083/current, will proceed with Du for space computation calculation, 2024-12-03T20:34:12,491 WARN [Thread-127 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/data/data4/current/BP-1199604065-172.17.0.3-1733258049083/current, will proceed with Du for space computation calculation, 2024-12-03T20:34:12,524 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T20:34:12,524 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T20:34:12,524 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T20:34:12,579 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd25d675d6325348 with lease ID 0x4e38d2972e359982: Processing first storage report for DS-92eaa216-f151-4266-97d7-3e835c8da57f from datanode DatanodeRegistration(127.0.0.1:36623, datanodeUuid=d1035785-19d2-46d8-8b47-e021c23353d0, infoPort=33459, infoSecurePort=0, ipcPort=39159, storageInfo=lv=-57;cid=testClusterID;nsid=2049534363;c=1733258049083) 2024-12-03T20:34:12,580 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd25d675d6325348 with lease ID 0x4e38d2972e359982: from storage DS-92eaa216-f151-4266-97d7-3e835c8da57f node DatanodeRegistration(127.0.0.1:36623, datanodeUuid=d1035785-19d2-46d8-8b47-e021c23353d0, infoPort=33459, infoSecurePort=0, ipcPort=39159, storageInfo=lv=-57;cid=testClusterID;nsid=2049534363;c=1733258049083), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T20:34:12,581 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x74b3129ea046601b with lease ID 0x4e38d2972e359984: Processing first storage report for DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae from datanode DatanodeRegistration(127.0.0.1:33321, datanodeUuid=0eb744f4-f008-4db8-8d56-69feb009b4cb, infoPort=36683, infoSecurePort=0, ipcPort=35521, storageInfo=lv=-57;cid=testClusterID;nsid=2049534363;c=1733258049083) 2024-12-03T20:34:12,581 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x74b3129ea046601b with lease ID 0x4e38d2972e359984: from storage DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae node DatanodeRegistration(127.0.0.1:33321, datanodeUuid=0eb744f4-f008-4db8-8d56-69feb009b4cb, infoPort=36683, infoSecurePort=0, ipcPort=35521, storageInfo=lv=-57;cid=testClusterID;nsid=2049534363;c=1733258049083), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T20:34:12,581 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2a76fe90dff18729 with lease ID 0x4e38d2972e359983: Processing first storage report for DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd from datanode DatanodeRegistration(127.0.0.1:46119, datanodeUuid=c3badc97-6773-498e-871f-361f9428500c, infoPort=34005, infoSecurePort=0, ipcPort=42295, storageInfo=lv=-57;cid=testClusterID;nsid=2049534363;c=1733258049083) 2024-12-03T20:34:12,581 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2a76fe90dff18729 with lease ID 0x4e38d2972e359983: from storage DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd node DatanodeRegistration(127.0.0.1:46119, datanodeUuid=c3badc97-6773-498e-871f-361f9428500c, infoPort=34005, infoSecurePort=0, ipcPort=42295, storageInfo=lv=-57;cid=testClusterID;nsid=2049534363;c=1733258049083), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T20:34:12,581 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd25d675d6325348 with lease ID 0x4e38d2972e359982: Processing first storage report for DS-d8854bf0-40f3-4d6f-987a-26ec871bcafc from datanode DatanodeRegistration(127.0.0.1:36623, datanodeUuid=d1035785-19d2-46d8-8b47-e021c23353d0, infoPort=33459, infoSecurePort=0, ipcPort=39159, storageInfo=lv=-57;cid=testClusterID;nsid=2049534363;c=1733258049083) 2024-12-03T20:34:12,582 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd25d675d6325348 with lease ID 0x4e38d2972e359982: from storage DS-d8854bf0-40f3-4d6f-987a-26ec871bcafc node DatanodeRegistration(127.0.0.1:36623, datanodeUuid=d1035785-19d2-46d8-8b47-e021c23353d0, infoPort=33459, infoSecurePort=0, ipcPort=39159, storageInfo=lv=-57;cid=testClusterID;nsid=2049534363;c=1733258049083), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T20:34:12,582 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x74b3129ea046601b with lease ID 0x4e38d2972e359984: Processing first storage report for DS-5c7dbcf6-3d1e-4900-a97b-36782188d4ab from datanode DatanodeRegistration(127.0.0.1:33321, datanodeUuid=0eb744f4-f008-4db8-8d56-69feb009b4cb, infoPort=36683, infoSecurePort=0, ipcPort=35521, storageInfo=lv=-57;cid=testClusterID;nsid=2049534363;c=1733258049083) 2024-12-03T20:34:12,582 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x74b3129ea046601b with lease ID 0x4e38d2972e359984: from storage DS-5c7dbcf6-3d1e-4900-a97b-36782188d4ab node DatanodeRegistration(127.0.0.1:33321, datanodeUuid=0eb744f4-f008-4db8-8d56-69feb009b4cb, infoPort=36683, infoSecurePort=0, ipcPort=35521, storageInfo=lv=-57;cid=testClusterID;nsid=2049534363;c=1733258049083), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T20:34:12,582 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2a76fe90dff18729 with lease ID 0x4e38d2972e359983: Processing first storage report for DS-a18d9fe1-569e-4255-a8ec-b07c7e1f8ce2 from datanode DatanodeRegistration(127.0.0.1:46119, datanodeUuid=c3badc97-6773-498e-871f-361f9428500c, infoPort=34005, infoSecurePort=0, ipcPort=42295, storageInfo=lv=-57;cid=testClusterID;nsid=2049534363;c=1733258049083) 2024-12-03T20:34:12,582 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2a76fe90dff18729 with lease ID 0x4e38d2972e359983: from storage DS-a18d9fe1-569e-4255-a8ec-b07c7e1f8ce2 node DatanodeRegistration(127.0.0.1:46119, datanodeUuid=c3badc97-6773-498e-871f-361f9428500c, infoPort=34005, infoSecurePort=0, ipcPort=42295, storageInfo=lv=-57;cid=testClusterID;nsid=2049534363;c=1733258049083), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T20:34:12,666 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7 2024-12-03T20:34:12,738 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/zookeeper_0, clientPort=49830, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T20:34:12,747 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49830 2024-12-03T20:34:12,762 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T20:34:12,765 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T20:34:12,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741825_1001 (size=7) 2024-12-03T20:34:12,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741825_1001 (size=7) 2024-12-03T20:34:12,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741825_1001 (size=7) 2024-12-03T20:34:13,362 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596 with version=8 2024-12-03T20:34:13,362 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/hbase-staging 2024-12-03T20:34:13,623 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b29c245002d9:0 server-side Connection retries=45 2024-12-03T20:34:13,631 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T20:34:13,631 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T20:34:13,636 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T20:34:13,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T20:34:13,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T20:34:13,743 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T20:34:13,790 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T20:34:13,797 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T20:34:13,800 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T20:34:13,821 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 67033 (auto-detected) 2024-12-03T20:34:13,822 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-03T20:34:13,838 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38205 2024-12-03T20:34:13,855 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38205 connecting to ZooKeeper ensemble=127.0.0.1:49830 2024-12-03T20:34:14,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:382050x0, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T20:34:14,006 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38205-0x1019ce700170000 connected 2024-12-03T20:34:14,110 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T20:34:14,116 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T20:34:14,129 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T20:34:14,132 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596, hbase.cluster.distributed=false 2024-12-03T20:34:14,153 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T20:34:14,158 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38205 2024-12-03T20:34:14,158 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38205 2024-12-03T20:34:14,160 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38205 2024-12-03T20:34:14,161 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38205 2024-12-03T20:34:14,164 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38205 2024-12-03T20:34:14,269 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b29c245002d9:0 server-side Connection retries=45 2024-12-03T20:34:14,271 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T20:34:14,271 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T20:34:14,271 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T20:34:14,272 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T20:34:14,272 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T20:34:14,274 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T20:34:14,276 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T20:34:14,277 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35249 2024-12-03T20:34:14,279 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35249 connecting to ZooKeeper ensemble=127.0.0.1:49830 2024-12-03T20:34:14,280 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T20:34:14,282 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T20:34:14,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:352490x0, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T20:34:14,297 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:352490x0, quorum=127.0.0.1:49830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T20:34:14,297 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35249-0x1019ce700170001 connected 2024-12-03T20:34:14,300 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T20:34:14,307 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T20:34:14,310 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35249-0x1019ce700170001, quorum=127.0.0.1:49830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T20:34:14,314 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35249-0x1019ce700170001, quorum=127.0.0.1:49830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T20:34:14,315 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35249 2024-12-03T20:34:14,315 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35249 2024-12-03T20:34:14,316 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35249 2024-12-03T20:34:14,316 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35249 2024-12-03T20:34:14,317 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35249 2024-12-03T20:34:14,331 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b29c245002d9:0 server-side Connection retries=45 2024-12-03T20:34:14,331 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T20:34:14,331 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T20:34:14,332 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T20:34:14,332 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T20:34:14,332 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T20:34:14,332 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T20:34:14,332 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T20:34:14,333 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44079 2024-12-03T20:34:14,334 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44079 connecting to ZooKeeper ensemble=127.0.0.1:49830 2024-12-03T20:34:14,335 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T20:34:14,337 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T20:34:14,349 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:440790x0, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T20:34:14,350 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:440790x0, quorum=127.0.0.1:49830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T20:34:14,350 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44079-0x1019ce700170002 connected 2024-12-03T20:34:14,350 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T20:34:14,351 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T20:34:14,352 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44079-0x1019ce700170002, quorum=127.0.0.1:49830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T20:34:14,354 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44079-0x1019ce700170002, quorum=127.0.0.1:49830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T20:34:14,355 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44079 2024-12-03T20:34:14,355 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44079 2024-12-03T20:34:14,355 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44079 2024-12-03T20:34:14,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44079 2024-12-03T20:34:14,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44079 2024-12-03T20:34:14,369 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b29c245002d9:0 server-side Connection retries=45 2024-12-03T20:34:14,369 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T20:34:14,369 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T20:34:14,370 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T20:34:14,370 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T20:34:14,370 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T20:34:14,370 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T20:34:14,370 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T20:34:14,371 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35007 2024-12-03T20:34:14,372 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35007 connecting to ZooKeeper ensemble=127.0.0.1:49830 2024-12-03T20:34:14,373 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T20:34:14,375 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T20:34:14,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:350070x0, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T20:34:14,402 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:350070x0, quorum=127.0.0.1:49830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T20:34:14,402 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35007-0x1019ce700170003 connected 2024-12-03T20:34:14,403 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T20:34:14,404 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T20:34:14,405 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35007-0x1019ce700170003, quorum=127.0.0.1:49830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T20:34:14,406 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35007-0x1019ce700170003, quorum=127.0.0.1:49830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T20:34:14,407 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35007 2024-12-03T20:34:14,407 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35007 2024-12-03T20:34:14,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35007 2024-12-03T20:34:14,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35007 2024-12-03T20:34:14,412 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35007 2024-12-03T20:34:14,425 DEBUG [M:0;b29c245002d9:38205 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b29c245002d9:38205 2024-12-03T20:34:14,426 INFO [master/b29c245002d9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b29c245002d9,38205,1733258053478 2024-12-03T20:34:14,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35007-0x1019ce700170003, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T20:34:14,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T20:34:14,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35249-0x1019ce700170001, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T20:34:14,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1019ce700170002, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T20:34:14,446 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b29c245002d9,38205,1733258053478 2024-12-03T20:34:14,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1019ce700170002, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T20:34:14,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:14,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35249-0x1019ce700170001, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T20:34:14,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1019ce700170002, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:14,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35007-0x1019ce700170003, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T20:34:14,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35249-0x1019ce700170001, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:14,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35007-0x1019ce700170003, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:14,476 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T20:34:14,477 INFO [master/b29c245002d9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b29c245002d9,38205,1733258053478 from backup master directory 2024-12-03T20:34:14,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b29c245002d9,38205,1733258053478 2024-12-03T20:34:14,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35007-0x1019ce700170003, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T20:34:14,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35249-0x1019ce700170001, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T20:34:14,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1019ce700170002, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T20:34:14,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T20:34:14,487 WARN [master/b29c245002d9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T20:34:14,487 INFO [master/b29c245002d9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b29c245002d9,38205,1733258053478 2024-12-03T20:34:14,490 INFO [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T20:34:14,492 INFO [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T20:34:14,543 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/hbase.id] with ID: 3811d7df-5f5c-4ed7-96e1-56c5b559222b 2024-12-03T20:34:14,544 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/.tmp/hbase.id 2024-12-03T20:34:14,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741826_1002 (size=42) 2024-12-03T20:34:14,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741826_1002 (size=42) 2024-12-03T20:34:14,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741826_1002 (size=42) 2024-12-03T20:34:14,558 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/.tmp/hbase.id]:[hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/hbase.id] 2024-12-03T20:34:14,599 INFO [master/b29c245002d9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T20:34:14,603 INFO [master/b29c245002d9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T20:34:14,620 INFO [master/b29c245002d9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-03T20:34:14,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35249-0x1019ce700170001, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:14,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35007-0x1019ce700170003, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:14,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:14,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1019ce700170002, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:14,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741827_1003 (size=196) 2024-12-03T20:34:14,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741827_1003 (size=196) 2024-12-03T20:34:14,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741827_1003 (size=196) 2024-12-03T20:34:14,671 INFO [master/b29c245002d9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T20:34:14,674 INFO [master/b29c245002d9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T20:34:14,681 INFO [master/b29c245002d9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:14,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741828_1004 (size=1189) 2024-12-03T20:34:14,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741828_1004 (size=1189) 2024-12-03T20:34:14,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741828_1004 (size=1189) 2024-12-03T20:34:14,725 INFO [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/data/master/store 2024-12-03T20:34:14,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741829_1005 (size=34) 2024-12-03T20:34:14,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741829_1005 (size=34) 2024-12-03T20:34:14,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741829_1005 (size=34) 2024-12-03T20:34:14,754 INFO [master/b29c245002d9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T20:34:14,758 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:14,759 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T20:34:14,759 INFO [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T20:34:14,760 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T20:34:14,761 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T20:34:14,762 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T20:34:14,762 INFO [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T20:34:14,763 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733258054759Disabling compacts and flushes for region at 1733258054759Disabling writes for close at 1733258054762 (+3 ms)Writing region close event to WAL at 1733258054762Closed at 1733258054762 2024-12-03T20:34:14,766 WARN [master/b29c245002d9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/data/master/store/.initializing 2024-12-03T20:34:14,766 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/WALs/b29c245002d9,38205,1733258053478 2024-12-03T20:34:14,774 INFO [master/b29c245002d9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T20:34:14,787 INFO [master/b29c245002d9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b29c245002d9%2C38205%2C1733258053478, suffix=, logDir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/WALs/b29c245002d9,38205,1733258053478, archiveDir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/oldWALs, maxLogs=10 2024-12-03T20:34:14,815 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/WALs/b29c245002d9,38205,1733258053478/b29c245002d9%2C38205%2C1733258053478.1733258054791, exclude list is [], retry=0 2024-12-03T20:34:14,833 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:14,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:14,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:14,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:14,842 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T20:34:14,880 INFO [master/b29c245002d9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/WALs/b29c245002d9,38205,1733258053478/b29c245002d9%2C38205%2C1733258053478.1733258054791 2024-12-03T20:34:14,881 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:34005:34005)] 2024-12-03T20:34:14,882 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:14,882 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:14,885 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T20:34:14,885 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T20:34:14,919 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T20:34:14,941 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T20:34:14,944 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:14,946 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T20:34:14,947 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T20:34:14,950 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T20:34:14,950 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:14,951 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:14,951 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T20:34:14,954 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T20:34:14,954 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:14,955 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:14,955 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T20:34:14,958 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T20:34:14,958 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:14,959 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:14,959 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T20:34:14,963 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T20:34:14,964 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T20:34:14,969 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T20:34:14,970 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T20:34:14,973 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T20:34:14,976 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T20:34:14,979 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T20:34:14,980 INFO [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62536321, jitterRate=-0.06813620030879974}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T20:34:14,986 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733258054896Initializing all the Stores at 1733258054898 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733258054899 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258054899Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258054900 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258054900Cleaning up temporary data from old regions at 1733258054970 (+70 ms)Region opened successfully at 1733258054986 (+16 ms) 2024-12-03T20:34:14,987 INFO [master/b29c245002d9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T20:34:15,016 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef1a435, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b29c245002d9/172.17.0.3:0 2024-12-03T20:34:15,041 INFO [master/b29c245002d9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T20:34:15,051 INFO [master/b29c245002d9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T20:34:15,051 INFO [master/b29c245002d9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T20:34:15,053 INFO [master/b29c245002d9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T20:34:15,054 INFO [master/b29c245002d9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T20:34:15,058 INFO [master/b29c245002d9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-03T20:34:15,059 INFO [master/b29c245002d9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T20:34:15,084 INFO [master/b29c245002d9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T20:34:15,092 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T20:34:15,159 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T20:34:15,163 INFO [master/b29c245002d9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T20:34:15,165 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T20:34:15,176 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T20:34:15,178 INFO [master/b29c245002d9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T20:34:15,181 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T20:34:15,191 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T20:34:15,192 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T20:34:15,201 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T20:34:15,223 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T20:34:15,233 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T20:34:15,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35007-0x1019ce700170003, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T20:34:15,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T20:34:15,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35249-0x1019ce700170001, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T20:34:15,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1019ce700170002, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T20:34:15,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35007-0x1019ce700170003, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:15,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:15,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35249-0x1019ce700170001, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:15,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1019ce700170002, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:15,248 INFO [master/b29c245002d9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b29c245002d9,38205,1733258053478, sessionid=0x1019ce700170000, setting cluster-up flag (Was=false) 2024-12-03T20:34:15,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1019ce700170002, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:15,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35007-0x1019ce700170003, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:15,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:15,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35249-0x1019ce700170001, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:15,402 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T20:34:15,407 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b29c245002d9,38205,1733258053478 2024-12-03T20:34:15,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35007-0x1019ce700170003, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:15,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1019ce700170002, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:15,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:15,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35249-0x1019ce700170001, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:15,465 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T20:34:15,469 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b29c245002d9,38205,1733258053478 2024-12-03T20:34:15,480 INFO [master/b29c245002d9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T20:34:15,516 INFO [RS:1;b29c245002d9:44079 {}] regionserver.HRegionServer(746): ClusterId : 3811d7df-5f5c-4ed7-96e1-56c5b559222b 2024-12-03T20:34:15,516 INFO [RS:2;b29c245002d9:35007 {}] regionserver.HRegionServer(746): ClusterId : 3811d7df-5f5c-4ed7-96e1-56c5b559222b 2024-12-03T20:34:15,516 INFO [RS:0;b29c245002d9:35249 {}] regionserver.HRegionServer(746): ClusterId : 3811d7df-5f5c-4ed7-96e1-56c5b559222b 2024-12-03T20:34:15,518 DEBUG [RS:2;b29c245002d9:35007 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T20:34:15,518 DEBUG [RS:1;b29c245002d9:44079 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T20:34:15,518 DEBUG [RS:0;b29c245002d9:35249 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T20:34:15,540 DEBUG [RS:2;b29c245002d9:35007 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T20:34:15,540 DEBUG [RS:1;b29c245002d9:44079 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T20:34:15,540 DEBUG [RS:0;b29c245002d9:35249 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T20:34:15,540 DEBUG [RS:2;b29c245002d9:35007 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T20:34:15,540 DEBUG [RS:1;b29c245002d9:44079 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T20:34:15,540 DEBUG [RS:0;b29c245002d9:35249 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T20:34:15,547 INFO [AsyncFSWAL-0-hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData-prefix:b29c245002d9,38205,1733258053478 {}] compress.Compression(560): Loaded codec org.apache.hadoop.hbase.io.compress.ReusableStreamGzipCodec for compression algorithm GZ 2024-12-03T20:34:15,550 DEBUG [RS:0;b29c245002d9:35249 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T20:34:15,550 DEBUG [RS:2;b29c245002d9:35007 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T20:34:15,550 DEBUG [RS:1;b29c245002d9:44079 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T20:34:15,550 DEBUG [RS:0;b29c245002d9:35249 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12a0330b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b29c245002d9/172.17.0.3:0 2024-12-03T20:34:15,550 DEBUG [RS:1;b29c245002d9:44079 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c54479b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b29c245002d9/172.17.0.3:0 2024-12-03T20:34:15,550 DEBUG [RS:2;b29c245002d9:35007 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aa62e14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b29c245002d9/172.17.0.3:0 2024-12-03T20:34:15,563 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T20:34:15,565 DEBUG [RS:2;b29c245002d9:35007 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;b29c245002d9:35007 2024-12-03T20:34:15,568 DEBUG [RS:0;b29c245002d9:35249 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b29c245002d9:35249 2024-12-03T20:34:15,568 INFO [RS:2;b29c245002d9:35007 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T20:34:15,568 INFO [RS:0;b29c245002d9:35249 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T20:34:15,568 INFO [RS:2;b29c245002d9:35007 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T20:34:15,568 INFO [RS:0;b29c245002d9:35249 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T20:34:15,568 DEBUG [RS:2;b29c245002d9:35007 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T20:34:15,568 DEBUG [RS:1;b29c245002d9:44079 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;b29c245002d9:44079 2024-12-03T20:34:15,569 DEBUG [RS:0;b29c245002d9:35249 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T20:34:15,569 INFO [RS:1;b29c245002d9:44079 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T20:34:15,569 INFO [RS:1;b29c245002d9:44079 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T20:34:15,569 DEBUG [RS:1;b29c245002d9:44079 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T20:34:15,571 INFO [RS:1;b29c245002d9:44079 {}] regionserver.HRegionServer(2659): reportForDuty to master=b29c245002d9,38205,1733258053478 with port=44079, startcode=1733258054330 2024-12-03T20:34:15,571 INFO [RS:0;b29c245002d9:35249 {}] regionserver.HRegionServer(2659): reportForDuty to master=b29c245002d9,38205,1733258053478 with port=35249, startcode=1733258054237 2024-12-03T20:34:15,571 INFO [RS:2;b29c245002d9:35007 {}] regionserver.HRegionServer(2659): reportForDuty to master=b29c245002d9,38205,1733258053478 with port=35007, startcode=1733258054368 2024-12-03T20:34:15,573 INFO [master/b29c245002d9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T20:34:15,579 INFO [master/b29c245002d9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T20:34:15,582 DEBUG [RS:1;b29c245002d9:44079 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T20:34:15,582 DEBUG [RS:0;b29c245002d9:35249 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T20:34:15,582 DEBUG [RS:2;b29c245002d9:35007 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T20:34:15,584 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b29c245002d9,38205,1733258053478 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T20:34:15,591 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b29c245002d9:0, corePoolSize=5, maxPoolSize=5 2024-12-03T20:34:15,591 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b29c245002d9:0, corePoolSize=5, maxPoolSize=5 2024-12-03T20:34:15,591 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b29c245002d9:0, corePoolSize=5, maxPoolSize=5 2024-12-03T20:34:15,592 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b29c245002d9:0, corePoolSize=5, maxPoolSize=5 2024-12-03T20:34:15,592 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b29c245002d9:0, corePoolSize=10, maxPoolSize=10 2024-12-03T20:34:15,592 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,592 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b29c245002d9:0, corePoolSize=2, maxPoolSize=2 2024-12-03T20:34:15,593 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,594 INFO [master/b29c245002d9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733258085594 2024-12-03T20:34:15,596 INFO [master/b29c245002d9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T20:34:15,598 INFO [master/b29c245002d9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T20:34:15,601 INFO [master/b29c245002d9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T20:34:15,602 INFO [master/b29c245002d9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T20:34:15,603 INFO [master/b29c245002d9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T20:34:15,603 INFO [master/b29c245002d9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T20:34:15,604 INFO [master/b29c245002d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,608 INFO [master/b29c245002d9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T20:34:15,609 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T20:34:15,609 INFO [master/b29c245002d9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T20:34:15,609 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T20:34:15,609 INFO [master/b29c245002d9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T20:34:15,611 INFO [master/b29c245002d9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T20:34:15,611 INFO [master/b29c245002d9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T20:34:15,614 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32845, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T20:34:15,614 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43385, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T20:34:15,614 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36635, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T20:34:15,614 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b29c245002d9:0:becomeActiveMaster-HFileCleaner.large.0-1733258055613,5,FailOnTimeoutGroup] 2024-12-03T20:34:15,616 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b29c245002d9:0:becomeActiveMaster-HFileCleaner.small.0-1733258055614,5,FailOnTimeoutGroup] 2024-12-03T20:34:15,616 INFO [master/b29c245002d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,616 INFO [master/b29c245002d9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T20:34:15,616 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:15,616 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T20:34:15,617 INFO [master/b29c245002d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,617 INFO [master/b29c245002d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,620 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38205 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b29c245002d9,35007,1733258054368 2024-12-03T20:34:15,623 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38205 {}] master.ServerManager(517): Registering regionserver=b29c245002d9,35007,1733258054368 2024-12-03T20:34:15,645 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38205 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b29c245002d9,44079,1733258054330 2024-12-03T20:34:15,645 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38205 {}] master.ServerManager(517): Registering regionserver=b29c245002d9,44079,1733258054330 2024-12-03T20:34:15,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741831_1007 (size=1321) 2024-12-03T20:34:15,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741831_1007 (size=1321) 2024-12-03T20:34:15,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741831_1007 (size=1321) 2024-12-03T20:34:15,649 DEBUG [RS:2;b29c245002d9:35007 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596 2024-12-03T20:34:15,649 DEBUG [RS:2;b29c245002d9:35007 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39361 2024-12-03T20:34:15,649 DEBUG [RS:2;b29c245002d9:35007 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T20:34:15,651 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38205 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b29c245002d9,35249,1733258054237 2024-12-03T20:34:15,651 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T20:34:15,651 DEBUG [RS:1;b29c245002d9:44079 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596 2024-12-03T20:34:15,651 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38205 {}] master.ServerManager(517): Registering regionserver=b29c245002d9,35249,1733258054237 2024-12-03T20:34:15,651 DEBUG [RS:1;b29c245002d9:44079 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39361 2024-12-03T20:34:15,651 DEBUG [RS:1;b29c245002d9:44079 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T20:34:15,651 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596 2024-12-03T20:34:15,655 DEBUG [RS:0;b29c245002d9:35249 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596 2024-12-03T20:34:15,655 DEBUG [RS:0;b29c245002d9:35249 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39361 2024-12-03T20:34:15,655 DEBUG [RS:0;b29c245002d9:35249 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T20:34:15,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741832_1008 (size=32) 2024-12-03T20:34:15,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741832_1008 (size=32) 2024-12-03T20:34:15,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741832_1008 (size=32) 2024-12-03T20:34:15,666 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:15,667 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T20:34:15,670 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T20:34:15,670 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:15,671 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T20:34:15,671 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T20:34:15,673 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T20:34:15,674 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:15,675 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T20:34:15,675 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T20:34:15,678 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T20:34:15,678 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:15,679 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T20:34:15,679 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T20:34:15,682 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T20:34:15,682 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:15,683 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T20:34:15,683 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T20:34:15,684 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740 2024-12-03T20:34:15,685 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740 2024-12-03T20:34:15,687 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T20:34:15,687 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T20:34:15,688 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T20:34:15,690 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T20:34:15,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T20:34:15,695 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T20:34:15,696 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74200498, jitterRate=0.10567358136177063}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T20:34:15,699 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733258055666Initializing all the Stores at 1733258055667 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733258055667Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733258055667Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258055667Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733258055667Cleaning up temporary data from old regions at 1733258055687 (+20 ms)Region opened successfully at 1733258055699 (+12 ms) 2024-12-03T20:34:15,700 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T20:34:15,700 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T20:34:15,700 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T20:34:15,700 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T20:34:15,700 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T20:34:15,701 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T20:34:15,701 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733258055699Disabling compacts and flushes for region at 1733258055699Disabling writes for close at 1733258055700 (+1 ms)Writing region close event to WAL at 1733258055701 (+1 ms)Closed at 1733258055701 2024-12-03T20:34:15,704 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T20:34:15,704 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T20:34:15,707 DEBUG [RS:1;b29c245002d9:44079 {}] zookeeper.ZKUtil(111): regionserver:44079-0x1019ce700170002, quorum=127.0.0.1:49830, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b29c245002d9,44079,1733258054330 2024-12-03T20:34:15,707 DEBUG [RS:2;b29c245002d9:35007 {}] zookeeper.ZKUtil(111): regionserver:35007-0x1019ce700170003, quorum=127.0.0.1:49830, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b29c245002d9,35007,1733258054368 2024-12-03T20:34:15,707 WARN [RS:1;b29c245002d9:44079 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T20:34:15,707 WARN [RS:2;b29c245002d9:35007 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T20:34:15,707 INFO [RS:2;b29c245002d9:35007 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:15,707 INFO [RS:1;b29c245002d9:44079 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:15,707 DEBUG [RS:0;b29c245002d9:35249 {}] zookeeper.ZKUtil(111): regionserver:35249-0x1019ce700170001, quorum=127.0.0.1:49830, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b29c245002d9,35249,1733258054237 2024-12-03T20:34:15,707 WARN [RS:0;b29c245002d9:35249 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T20:34:15,707 DEBUG [RS:1;b29c245002d9:44079 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,44079,1733258054330 2024-12-03T20:34:15,707 DEBUG [RS:2;b29c245002d9:35007 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35007,1733258054368 2024-12-03T20:34:15,707 INFO [RS:0;b29c245002d9:35249 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:15,708 DEBUG [RS:0;b29c245002d9:35249 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35249,1733258054237 2024-12-03T20:34:15,709 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b29c245002d9,35007,1733258054368] 2024-12-03T20:34:15,709 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b29c245002d9,35249,1733258054237] 2024-12-03T20:34:15,709 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b29c245002d9,44079,1733258054330] 2024-12-03T20:34:15,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T20:34:15,721 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T20:34:15,724 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T20:34:15,734 INFO [RS:0;b29c245002d9:35249 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T20:34:15,734 INFO [RS:2;b29c245002d9:35007 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T20:34:15,734 INFO [RS:1;b29c245002d9:44079 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T20:34:15,747 INFO [RS:2;b29c245002d9:35007 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T20:34:15,747 INFO [RS:0;b29c245002d9:35249 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T20:34:15,747 INFO [RS:1;b29c245002d9:44079 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T20:34:15,752 INFO [RS:2;b29c245002d9:35007 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T20:34:15,753 INFO [RS:2;b29c245002d9:35007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,753 INFO [RS:1;b29c245002d9:44079 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T20:34:15,753 INFO [RS:0;b29c245002d9:35249 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T20:34:15,754 INFO [RS:1;b29c245002d9:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,754 INFO [RS:0;b29c245002d9:35249 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,754 INFO [RS:2;b29c245002d9:35007 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T20:34:15,754 INFO [RS:1;b29c245002d9:44079 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T20:34:15,754 INFO [RS:0;b29c245002d9:35249 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T20:34:15,761 INFO [RS:0;b29c245002d9:35249 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T20:34:15,761 INFO [RS:2;b29c245002d9:35007 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T20:34:15,761 INFO [RS:1;b29c245002d9:44079 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T20:34:15,762 INFO [RS:2;b29c245002d9:35007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,762 INFO [RS:0;b29c245002d9:35249 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,762 INFO [RS:1;b29c245002d9:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,763 DEBUG [RS:2;b29c245002d9:35007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,763 DEBUG [RS:0;b29c245002d9:35249 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,763 DEBUG [RS:1;b29c245002d9:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,763 DEBUG [RS:2;b29c245002d9:35007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,763 DEBUG [RS:0;b29c245002d9:35249 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,763 DEBUG [RS:1;b29c245002d9:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,763 DEBUG [RS:2;b29c245002d9:35007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,763 DEBUG [RS:0;b29c245002d9:35249 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,763 DEBUG [RS:2;b29c245002d9:35007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,763 DEBUG [RS:1;b29c245002d9:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,763 DEBUG [RS:0;b29c245002d9:35249 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,763 DEBUG [RS:0;b29c245002d9:35249 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,763 DEBUG [RS:1;b29c245002d9:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,763 DEBUG [RS:2;b29c245002d9:35007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,763 DEBUG [RS:0;b29c245002d9:35249 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b29c245002d9:0, corePoolSize=2, maxPoolSize=2 2024-12-03T20:34:15,763 DEBUG [RS:2;b29c245002d9:35007 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b29c245002d9:0, corePoolSize=2, maxPoolSize=2 2024-12-03T20:34:15,763 DEBUG [RS:1;b29c245002d9:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,763 DEBUG [RS:0;b29c245002d9:35249 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,763 DEBUG [RS:2;b29c245002d9:35007 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,763 DEBUG [RS:0;b29c245002d9:35249 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,763 DEBUG [RS:1;b29c245002d9:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b29c245002d9:0, corePoolSize=2, maxPoolSize=2 2024-12-03T20:34:15,763 DEBUG [RS:0;b29c245002d9:35249 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,763 DEBUG [RS:2;b29c245002d9:35007 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,764 DEBUG [RS:1;b29c245002d9:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,764 DEBUG [RS:0;b29c245002d9:35249 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,764 DEBUG [RS:2;b29c245002d9:35007 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,764 DEBUG [RS:0;b29c245002d9:35249 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,764 DEBUG [RS:1;b29c245002d9:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,764 DEBUG [RS:0;b29c245002d9:35249 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,764 DEBUG [RS:2;b29c245002d9:35007 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,764 DEBUG [RS:1;b29c245002d9:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,764 DEBUG [RS:0;b29c245002d9:35249 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b29c245002d9:0, corePoolSize=3, maxPoolSize=3 2024-12-03T20:34:15,764 DEBUG [RS:2;b29c245002d9:35007 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,764 DEBUG [RS:1;b29c245002d9:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,764 DEBUG [RS:0;b29c245002d9:35249 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b29c245002d9:0, corePoolSize=3, maxPoolSize=3 2024-12-03T20:34:15,764 DEBUG [RS:2;b29c245002d9:35007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,764 DEBUG [RS:1;b29c245002d9:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,764 DEBUG [RS:2;b29c245002d9:35007 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b29c245002d9:0, corePoolSize=3, maxPoolSize=3 2024-12-03T20:34:15,764 DEBUG [RS:1;b29c245002d9:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b29c245002d9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T20:34:15,764 DEBUG [RS:2;b29c245002d9:35007 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b29c245002d9:0, corePoolSize=3, maxPoolSize=3 2024-12-03T20:34:15,764 DEBUG [RS:1;b29c245002d9:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b29c245002d9:0, corePoolSize=3, maxPoolSize=3 2024-12-03T20:34:15,765 DEBUG [RS:1;b29c245002d9:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b29c245002d9:0, corePoolSize=3, maxPoolSize=3 2024-12-03T20:34:15,766 INFO [RS:0;b29c245002d9:35249 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,766 INFO [RS:0;b29c245002d9:35249 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,766 INFO [RS:2;b29c245002d9:35007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,766 INFO [RS:0;b29c245002d9:35249 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,766 INFO [RS:2;b29c245002d9:35007 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,766 INFO [RS:1;b29c245002d9:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,766 INFO [RS:0;b29c245002d9:35249 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,767 INFO [RS:2;b29c245002d9:35007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,767 INFO [RS:1;b29c245002d9:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,767 INFO [RS:0;b29c245002d9:35249 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,767 INFO [RS:2;b29c245002d9:35007 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,767 INFO [RS:1;b29c245002d9:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,767 INFO [RS:0;b29c245002d9:35249 {}] hbase.ChoreService(168): Chore ScheduledChore name=b29c245002d9,35249,1733258054237-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T20:34:15,767 INFO [RS:1;b29c245002d9:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,767 INFO [RS:1;b29c245002d9:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,767 INFO [RS:1;b29c245002d9:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=b29c245002d9,44079,1733258054330-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T20:34:15,767 INFO [RS:2;b29c245002d9:35007 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,767 INFO [RS:2;b29c245002d9:35007 {}] hbase.ChoreService(168): Chore ScheduledChore name=b29c245002d9,35007,1733258054368-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T20:34:15,784 INFO [RS:0;b29c245002d9:35249 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T20:34:15,786 INFO [RS:0;b29c245002d9:35249 {}] hbase.ChoreService(168): Chore ScheduledChore name=b29c245002d9,35249,1733258054237-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,786 INFO [RS:0;b29c245002d9:35249 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,786 INFO [RS:0;b29c245002d9:35249 {}] regionserver.Replication(171): b29c245002d9,35249,1733258054237 started 2024-12-03T20:34:15,789 INFO [RS:1;b29c245002d9:44079 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T20:34:15,789 INFO [RS:2;b29c245002d9:35007 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T20:34:15,789 INFO [RS:1;b29c245002d9:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=b29c245002d9,44079,1733258054330-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,789 INFO [RS:2;b29c245002d9:35007 {}] hbase.ChoreService(168): Chore ScheduledChore name=b29c245002d9,35007,1733258054368-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,789 INFO [RS:1;b29c245002d9:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,789 INFO [RS:1;b29c245002d9:44079 {}] regionserver.Replication(171): b29c245002d9,44079,1733258054330 started 2024-12-03T20:34:15,789 INFO [RS:2;b29c245002d9:35007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,790 INFO [RS:2;b29c245002d9:35007 {}] regionserver.Replication(171): b29c245002d9,35007,1733258054368 started 2024-12-03T20:34:15,801 INFO [RS:0;b29c245002d9:35249 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,802 INFO [RS:0;b29c245002d9:35249 {}] regionserver.HRegionServer(1482): Serving as b29c245002d9,35249,1733258054237, RpcServer on b29c245002d9/172.17.0.3:35249, sessionid=0x1019ce700170001 2024-12-03T20:34:15,802 DEBUG [RS:0;b29c245002d9:35249 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T20:34:15,802 DEBUG [RS:0;b29c245002d9:35249 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b29c245002d9,35249,1733258054237 2024-12-03T20:34:15,802 DEBUG [RS:0;b29c245002d9:35249 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b29c245002d9,35249,1733258054237' 2024-12-03T20:34:15,803 DEBUG [RS:0;b29c245002d9:35249 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T20:34:15,803 DEBUG [RS:0;b29c245002d9:35249 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T20:34:15,804 DEBUG [RS:0;b29c245002d9:35249 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T20:34:15,804 DEBUG [RS:0;b29c245002d9:35249 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T20:34:15,804 DEBUG [RS:0;b29c245002d9:35249 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b29c245002d9,35249,1733258054237 2024-12-03T20:34:15,804 DEBUG [RS:0;b29c245002d9:35249 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b29c245002d9,35249,1733258054237' 2024-12-03T20:34:15,804 DEBUG [RS:0;b29c245002d9:35249 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T20:34:15,805 DEBUG [RS:0;b29c245002d9:35249 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T20:34:15,806 DEBUG [RS:0;b29c245002d9:35249 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T20:34:15,806 INFO [RS:0;b29c245002d9:35249 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T20:34:15,806 INFO [RS:0;b29c245002d9:35249 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T20:34:15,810 INFO [RS:2;b29c245002d9:35007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,810 INFO [RS:1;b29c245002d9:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:15,810 INFO [RS:2;b29c245002d9:35007 {}] regionserver.HRegionServer(1482): Serving as b29c245002d9,35007,1733258054368, RpcServer on b29c245002d9/172.17.0.3:35007, sessionid=0x1019ce700170003 2024-12-03T20:34:15,810 INFO [RS:1;b29c245002d9:44079 {}] regionserver.HRegionServer(1482): Serving as b29c245002d9,44079,1733258054330, RpcServer on b29c245002d9/172.17.0.3:44079, sessionid=0x1019ce700170002 2024-12-03T20:34:15,810 DEBUG [RS:2;b29c245002d9:35007 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T20:34:15,810 DEBUG [RS:1;b29c245002d9:44079 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T20:34:15,810 DEBUG [RS:2;b29c245002d9:35007 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b29c245002d9,35007,1733258054368 2024-12-03T20:34:15,810 DEBUG [RS:1;b29c245002d9:44079 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b29c245002d9,44079,1733258054330 2024-12-03T20:34:15,810 DEBUG [RS:2;b29c245002d9:35007 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b29c245002d9,35007,1733258054368' 2024-12-03T20:34:15,810 DEBUG [RS:1;b29c245002d9:44079 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b29c245002d9,44079,1733258054330' 2024-12-03T20:34:15,810 DEBUG [RS:2;b29c245002d9:35007 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T20:34:15,810 DEBUG [RS:1;b29c245002d9:44079 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T20:34:15,811 DEBUG [RS:2;b29c245002d9:35007 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T20:34:15,811 DEBUG [RS:1;b29c245002d9:44079 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T20:34:15,812 DEBUG [RS:2;b29c245002d9:35007 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T20:34:15,812 DEBUG [RS:1;b29c245002d9:44079 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T20:34:15,812 DEBUG [RS:2;b29c245002d9:35007 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T20:34:15,812 DEBUG [RS:1;b29c245002d9:44079 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T20:34:15,812 DEBUG [RS:2;b29c245002d9:35007 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b29c245002d9,35007,1733258054368 2024-12-03T20:34:15,812 DEBUG [RS:2;b29c245002d9:35007 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b29c245002d9,35007,1733258054368' 2024-12-03T20:34:15,812 DEBUG [RS:1;b29c245002d9:44079 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b29c245002d9,44079,1733258054330 2024-12-03T20:34:15,812 DEBUG [RS:2;b29c245002d9:35007 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T20:34:15,812 DEBUG [RS:1;b29c245002d9:44079 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b29c245002d9,44079,1733258054330' 2024-12-03T20:34:15,812 DEBUG [RS:1;b29c245002d9:44079 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T20:34:15,812 DEBUG [RS:2;b29c245002d9:35007 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T20:34:15,813 DEBUG [RS:1;b29c245002d9:44079 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T20:34:15,813 DEBUG [RS:2;b29c245002d9:35007 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T20:34:15,813 INFO [RS:2;b29c245002d9:35007 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T20:34:15,813 INFO [RS:2;b29c245002d9:35007 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T20:34:15,813 DEBUG [RS:1;b29c245002d9:44079 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T20:34:15,813 INFO [RS:1;b29c245002d9:44079 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T20:34:15,813 INFO [RS:1;b29c245002d9:44079 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T20:34:15,875 WARN [b29c245002d9:38205 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T20:34:15,913 INFO [RS:0;b29c245002d9:35249 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T20:34:15,914 INFO [RS:2;b29c245002d9:35007 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T20:34:15,914 INFO [RS:1;b29c245002d9:44079 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T20:34:15,916 INFO [RS:0;b29c245002d9:35249 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b29c245002d9%2C35249%2C1733258054237, suffix=, logDir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35249,1733258054237, archiveDir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/oldWALs, maxLogs=32 2024-12-03T20:34:15,917 INFO [RS:2;b29c245002d9:35007 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b29c245002d9%2C35007%2C1733258054368, suffix=, logDir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35007,1733258054368, archiveDir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/oldWALs, maxLogs=32 2024-12-03T20:34:15,917 INFO [RS:1;b29c245002d9:44079 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b29c245002d9%2C44079%2C1733258054330, suffix=, logDir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,44079,1733258054330, archiveDir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/oldWALs, maxLogs=32 2024-12-03T20:34:15,933 DEBUG [RS:0;b29c245002d9:35249 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35249,1733258054237/b29c245002d9%2C35249%2C1733258054237.1733258055920, exclude list is [], retry=0 2024-12-03T20:34:15,934 DEBUG [RS:1;b29c245002d9:44079 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,44079,1733258054330/b29c245002d9%2C44079%2C1733258054330.1733258055920, exclude list is [], retry=0 2024-12-03T20:34:15,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:15,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:15,937 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:15,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:15,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:15,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:15,940 DEBUG [RS:2;b29c245002d9:35007 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35007,1733258054368/b29c245002d9%2C35007%2C1733258054368.1733258055921, exclude list is [], retry=0 2024-12-03T20:34:15,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:15,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:15,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:15,946 INFO [RS:0;b29c245002d9:35249 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35249,1733258054237/b29c245002d9%2C35249%2C1733258054237.1733258055920 2024-12-03T20:34:15,948 DEBUG [RS:0;b29c245002d9:35249 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:36683:36683)] 2024-12-03T20:34:15,950 INFO [RS:2;b29c245002d9:35007 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35007,1733258054368/b29c245002d9%2C35007%2C1733258054368.1733258055921 2024-12-03T20:34:15,950 INFO [RS:1;b29c245002d9:44079 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,44079,1733258054330/b29c245002d9%2C44079%2C1733258054330.1733258055920 2024-12-03T20:34:15,950 DEBUG [RS:2;b29c245002d9:35007 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:34005:34005)] 2024-12-03T20:34:15,951 DEBUG [RS:1;b29c245002d9:44079 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:33459:33459)] 2024-12-03T20:34:16,128 DEBUG [b29c245002d9:38205 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-03T20:34:16,139 DEBUG [b29c245002d9:38205 {}] balancer.BalancerClusterState(204): Hosts are {b29c245002d9=0} racks are {/default-rack=0} 2024-12-03T20:34:16,146 DEBUG [b29c245002d9:38205 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T20:34:16,147 DEBUG [b29c245002d9:38205 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T20:34:16,147 DEBUG [b29c245002d9:38205 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-03T20:34:16,147 DEBUG [b29c245002d9:38205 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T20:34:16,147 DEBUG [b29c245002d9:38205 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T20:34:16,147 DEBUG [b29c245002d9:38205 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-03T20:34:16,147 INFO [b29c245002d9:38205 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T20:34:16,147 INFO [b29c245002d9:38205 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T20:34:16,147 INFO [b29c245002d9:38205 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-03T20:34:16,147 DEBUG [b29c245002d9:38205 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T20:34:16,154 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b29c245002d9,35249,1733258054237 2024-12-03T20:34:16,158 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b29c245002d9,35249,1733258054237, state=OPENING 2024-12-03T20:34:16,209 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T20:34:16,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:16,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1019ce700170002, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:16,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35007-0x1019ce700170003, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:16,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35249-0x1019ce700170001, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:16,226 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T20:34:16,226 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T20:34:16,226 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T20:34:16,227 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T20:34:16,229 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T20:34:16,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b29c245002d9,35249,1733258054237}] 2024-12-03T20:34:16,408 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T20:34:16,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48163, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T20:34:16,421 INFO [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T20:34:16,421 INFO [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:16,422 INFO [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T20:34:16,425 INFO [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b29c245002d9%2C35249%2C1733258054237.meta, suffix=.meta, logDir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35249,1733258054237, archiveDir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/oldWALs, maxLogs=32 2024-12-03T20:34:16,437 DEBUG [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35249,1733258054237/b29c245002d9%2C35249%2C1733258054237.meta.1733258056426.meta, exclude list is [], retry=0 2024-12-03T20:34:16,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:16,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:16,441 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:16,444 INFO [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35249,1733258054237/b29c245002d9%2C35249%2C1733258054237.meta.1733258056426.meta 2024-12-03T20:34:16,444 DEBUG [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:36683:36683)] 2024-12-03T20:34:16,444 DEBUG [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:16,446 DEBUG [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T20:34:16,448 DEBUG [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T20:34:16,452 INFO [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T20:34:16,456 DEBUG [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T20:34:16,456 DEBUG [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:16,457 DEBUG [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T20:34:16,457 DEBUG [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T20:34:16,459 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T20:34:16,461 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T20:34:16,461 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:16,462 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T20:34:16,462 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T20:34:16,463 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T20:34:16,463 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:16,464 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T20:34:16,464 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T20:34:16,465 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T20:34:16,465 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:16,466 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T20:34:16,466 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T20:34:16,467 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T20:34:16,467 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:16,468 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T20:34:16,468 DEBUG [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T20:34:16,470 DEBUG [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740 2024-12-03T20:34:16,472 DEBUG [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740 2024-12-03T20:34:16,474 DEBUG [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T20:34:16,475 DEBUG [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T20:34:16,475 DEBUG [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T20:34:16,478 DEBUG [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T20:34:16,479 INFO [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63504508, jitterRate=-0.05370908975601196}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T20:34:16,479 DEBUG [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T20:34:16,481 DEBUG [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733258056457Writing region info on filesystem at 1733258056457Initializing all the Stores at 1733258056459 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733258056459Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733258056459Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258056459Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733258056459Cleaning up temporary data from old regions at 1733258056475 (+16 ms)Running coprocessor post-open hooks at 1733258056479 (+4 ms)Region opened successfully at 1733258056481 (+2 ms) 2024-12-03T20:34:16,487 INFO [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733258056400 2024-12-03T20:34:16,498 DEBUG [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T20:34:16,498 INFO [RS_OPEN_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T20:34:16,501 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b29c245002d9,35249,1733258054237 2024-12-03T20:34:16,503 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b29c245002d9,35249,1733258054237, state=OPEN 2024-12-03T20:34:16,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T20:34:16,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1019ce700170002, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T20:34:16,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35249-0x1019ce700170001, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T20:34:16,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35007-0x1019ce700170003, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T20:34:16,513 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T20:34:16,514 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T20:34:16,514 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T20:34:16,514 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T20:34:16,514 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b29c245002d9,35249,1733258054237 2024-12-03T20:34:16,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T20:34:16,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b29c245002d9,35249,1733258054237 in 282 msec 2024-12-03T20:34:16,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T20:34:16,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 812 msec 2024-12-03T20:34:16,531 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T20:34:16,531 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T20:34:16,549 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T20:34:16,550 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b29c245002d9,35249,1733258054237, seqNum=-1] 2024-12-03T20:34:16,567 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T20:34:16,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57905, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T20:34:16,599 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0920 sec 2024-12-03T20:34:16,600 INFO [master/b29c245002d9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733258056599, completionTime=-1 2024-12-03T20:34:16,602 INFO [master/b29c245002d9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-03T20:34:16,602 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T20:34:16,648 INFO [master/b29c245002d9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-03T20:34:16,649 INFO [master/b29c245002d9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733258116649 2024-12-03T20:34:16,649 INFO [master/b29c245002d9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733258176649 2024-12-03T20:34:16,649 INFO [master/b29c245002d9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 46 msec 2024-12-03T20:34:16,651 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-03T20:34:16,663 INFO [master/b29c245002d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b29c245002d9,38205,1733258053478-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:16,664 INFO [master/b29c245002d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b29c245002d9,38205,1733258053478-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:16,664 INFO [master/b29c245002d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b29c245002d9,38205,1733258053478-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:16,666 INFO [master/b29c245002d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b29c245002d9:38205, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:16,666 INFO [master/b29c245002d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:16,667 INFO [master/b29c245002d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:16,674 DEBUG [master/b29c245002d9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T20:34:16,696 INFO [master/b29c245002d9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.208sec 2024-12-03T20:34:16,698 INFO [master/b29c245002d9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T20:34:16,699 INFO [master/b29c245002d9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T20:34:16,700 INFO [master/b29c245002d9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T20:34:16,701 INFO [master/b29c245002d9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T20:34:16,701 INFO [master/b29c245002d9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T20:34:16,702 INFO [master/b29c245002d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b29c245002d9,38205,1733258053478-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T20:34:16,702 INFO [master/b29c245002d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b29c245002d9,38205,1733258053478-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T20:34:16,707 DEBUG [master/b29c245002d9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T20:34:16,707 INFO [master/b29c245002d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T20:34:16,708 INFO [master/b29c245002d9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b29c245002d9,38205,1733258053478-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:16,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ddf6023, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T20:34:16,738 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b29c245002d9,38205,-1 for getting cluster id 2024-12-03T20:34:16,740 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T20:34:16,749 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3811d7df-5f5c-4ed7-96e1-56c5b559222b' 2024-12-03T20:34:16,752 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T20:34:16,752 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3811d7df-5f5c-4ed7-96e1-56c5b559222b" 2024-12-03T20:34:16,752 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f753370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T20:34:16,752 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b29c245002d9,38205,-1] 2024-12-03T20:34:16,755 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T20:34:16,756 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T20:34:16,757 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T20:34:16,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65c83e31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T20:34:16,760 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T20:34:16,766 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b29c245002d9,35249,1733258054237, seqNum=-1] 2024-12-03T20:34:16,767 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T20:34:16,769 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T20:34:16,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b29c245002d9,38205,1733258053478 2024-12-03T20:34:16,788 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:39361/hbase 2024-12-03T20:34:16,802 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=370, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=297, ProcessCount=11, AvailableMemoryMB=2428 2024-12-03T20:34:16,821 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:16,824 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:16,825 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T20:34:16,830 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-73491205, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/hregion-73491205, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:16,842 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-73491205/hregion-73491205.1733258056831, exclude list is [], retry=0 2024-12-03T20:34:16,847 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:16,847 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:16,847 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:16,850 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-73491205/hregion-73491205.1733258056831 2024-12-03T20:34:16,850 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:36683:36683)] 2024-12-03T20:34:16,851 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 76530ff022f88fcb2ea00d9ba12621f7, NAME => 'testReplayEditsWrittenIntoWAL,,1733258056822.76530ff022f88fcb2ea00d9ba12621f7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39361/hbase 2024-12-03T20:34:16,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741838_1014 (size=64) 2024-12-03T20:34:16,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741838_1014 (size=64) 2024-12-03T20:34:16,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741838_1014 (size=64) 2024-12-03T20:34:16,865 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733258056822.76530ff022f88fcb2ea00d9ba12621f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:16,867 INFO [StoreOpener-76530ff022f88fcb2ea00d9ba12621f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 76530ff022f88fcb2ea00d9ba12621f7 2024-12-03T20:34:16,869 INFO [StoreOpener-76530ff022f88fcb2ea00d9ba12621f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76530ff022f88fcb2ea00d9ba12621f7 columnFamilyName a 2024-12-03T20:34:16,869 DEBUG [StoreOpener-76530ff022f88fcb2ea00d9ba12621f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:16,870 INFO [StoreOpener-76530ff022f88fcb2ea00d9ba12621f7-1 {}] regionserver.HStore(327): Store=76530ff022f88fcb2ea00d9ba12621f7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:16,870 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 76530ff022f88fcb2ea00d9ba12621f7 2024-12-03T20:34:16,872 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7 2024-12-03T20:34:16,873 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7 2024-12-03T20:34:16,873 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 76530ff022f88fcb2ea00d9ba12621f7 2024-12-03T20:34:16,873 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 76530ff022f88fcb2ea00d9ba12621f7 2024-12-03T20:34:16,877 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 76530ff022f88fcb2ea00d9ba12621f7 2024-12-03T20:34:16,881 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T20:34:16,882 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 76530ff022f88fcb2ea00d9ba12621f7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73232957, jitterRate=0.09125609695911407}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T20:34:16,882 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 76530ff022f88fcb2ea00d9ba12621f7: Writing region info on filesystem at 1733258056865Initializing all the Stores at 1733258056866 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258056866Cleaning up temporary data from old regions at 1733258056873 (+7 ms)Region opened successfully at 1733258056882 (+9 ms) 2024-12-03T20:34:16,883 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 76530ff022f88fcb2ea00d9ba12621f7, disabling compactions & flushes 2024-12-03T20:34:16,883 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733258056822.76530ff022f88fcb2ea00d9ba12621f7. 2024-12-03T20:34:16,883 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733258056822.76530ff022f88fcb2ea00d9ba12621f7. 2024-12-03T20:34:16,883 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733258056822.76530ff022f88fcb2ea00d9ba12621f7. after waiting 0 ms 2024-12-03T20:34:16,883 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733258056822.76530ff022f88fcb2ea00d9ba12621f7. 2024-12-03T20:34:16,883 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733258056822.76530ff022f88fcb2ea00d9ba12621f7. 2024-12-03T20:34:16,883 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 76530ff022f88fcb2ea00d9ba12621f7: Waiting for close lock at 1733258056882Disabling compacts and flushes for region at 1733258056882Disabling writes for close at 1733258056883 (+1 ms)Writing region close event to WAL at 1733258056883Closed at 1733258056883 2024-12-03T20:34:16,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741837_1013 (size=95) 2024-12-03T20:34:16,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741837_1013 (size=95) 2024-12-03T20:34:16,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741837_1013 (size=95) 2024-12-03T20:34:16,895 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T20:34:16,895 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-73491205:(num 1733258056831) 2024-12-03T20:34:16,897 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T20:34:16,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741839_1015 (size=320) 2024-12-03T20:34:16,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741839_1015 (size=320) 2024-12-03T20:34:16,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741839_1015 (size=320) 2024-12-03T20:34:16,921 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T20:34:16,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741840_1016 (size=253) 2024-12-03T20:34:16,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741840_1016 (size=253) 2024-12-03T20:34:16,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741840_1016 (size=253) 2024-12-03T20:34:16,954 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-1, size=320 (320bytes) 2024-12-03T20:34:16,955 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-03T20:34:16,955 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-03T20:34:16,955 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-1 2024-12-03T20:34:16,961 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-1 after 3ms 2024-12-03T20:34:16,967 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:16,968 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-1 took 15ms 2024-12-03T20:34:16,977 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-1 so closing down 2024-12-03T20:34:16,978 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T20:34:16,980 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-03T20:34:16,982 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/recovered.edits/0000000000000000001-wal-1.temp 2024-12-03T20:34:16,983 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T20:34:16,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741841_1017 (size=320) 2024-12-03T20:34:16,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741841_1017 (size=320) 2024-12-03T20:34:16,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741841_1017 (size=320) 2024-12-03T20:34:16,994 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T20:34:16,997 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/recovered.edits/0000000000000000002 2024-12-03T20:34:17,001 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 28 ms; skipped=0; WAL=hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-12-03T20:34:17,001 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-1, journal: Splitting hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-1, size=320 (320bytes) at 1733258056954Finishing writing output for hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-1 so closing down at 1733258056977 (+23 ms)Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/recovered.edits/0000000000000000001-wal-1.temp at 1733258056982 (+5 ms)3 split writer threads finished at 1733258056983 (+1 ms)Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733258056994 (+11 ms)Rename recovered edits hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/recovered.edits/0000000000000000002 at 1733258056997 (+3 ms)Processed 2 edits across 1 Regions in 28 ms; skipped=0; WAL=hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1733258057001 (+4 ms) 2024-12-03T20:34:17,013 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-2, size=253 (253bytes) 2024-12-03T20:34:17,013 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-2 2024-12-03T20:34:17,014 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-2 after 1ms 2024-12-03T20:34:17,018 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:17,018 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-2 took 5ms 2024-12-03T20:34:17,020 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-2 so closing down 2024-12-03T20:34:17,020 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T20:34:17,023 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-03T20:34:17,024 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/recovered.edits/0000000000000000002-wal-2.temp 2024-12-03T20:34:17,025 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T20:34:17,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741842_1018 (size=253) 2024-12-03T20:34:17,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741842_1018 (size=253) 2024-12-03T20:34:17,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741842_1018 (size=253) 2024-12-03T20:34:17,034 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T20:34:17,038 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:17,041 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/recovered.edits/0000000000000000002-wal-2.temp, length=253 2024-12-03T20:34:17,042 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-12-03T20:34:17,043 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-2, journal: Splitting hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-2, size=253 (253bytes) at 1733258057013Finishing writing output for hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-2 so closing down at 1733258057020 (+7 ms)Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/recovered.edits/0000000000000000002-wal-2.temp at 1733258057024 (+4 ms)3 split writer threads finished at 1733258057025 (+1 ms)Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733258057034 (+9 ms)Processed 1 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1733258057043 (+9 ms) 2024-12-03T20:34:17,043 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T20:34:17,045 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:17,058 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal.1733258057046, exclude list is [], retry=0 2024-12-03T20:34:17,062 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:17,062 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:17,063 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:17,065 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal.1733258057046 2024-12-03T20:34:17,066 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:34005:34005)] 2024-12-03T20:34:17,066 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 76530ff022f88fcb2ea00d9ba12621f7, NAME => 'testReplayEditsWrittenIntoWAL,,1733258056822.76530ff022f88fcb2ea00d9ba12621f7.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:17,066 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733258056822.76530ff022f88fcb2ea00d9ba12621f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:17,066 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 76530ff022f88fcb2ea00d9ba12621f7 2024-12-03T20:34:17,066 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 76530ff022f88fcb2ea00d9ba12621f7 2024-12-03T20:34:17,068 INFO [StoreOpener-76530ff022f88fcb2ea00d9ba12621f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 76530ff022f88fcb2ea00d9ba12621f7 2024-12-03T20:34:17,069 INFO [StoreOpener-76530ff022f88fcb2ea00d9ba12621f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76530ff022f88fcb2ea00d9ba12621f7 columnFamilyName a 2024-12-03T20:34:17,070 DEBUG [StoreOpener-76530ff022f88fcb2ea00d9ba12621f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:17,071 INFO [StoreOpener-76530ff022f88fcb2ea00d9ba12621f7-1 {}] regionserver.HStore(327): Store=76530ff022f88fcb2ea00d9ba12621f7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:17,071 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 76530ff022f88fcb2ea00d9ba12621f7 2024-12-03T20:34:17,072 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7 2024-12-03T20:34:17,075 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7 2024-12-03T20:34:17,076 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/recovered.edits/0000000000000000002 2024-12-03T20:34:17,079 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:17,085 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/recovered.edits/0000000000000000002 2024-12-03T20:34:17,088 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 76530ff022f88fcb2ea00d9ba12621f7 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-03T20:34:17,131 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/.tmp/a/a2460504cc55496d9c49cb33ba994307 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733258056895/Put/seqid=0 2024-12-03T20:34:17,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741844_1020 (size=5170) 2024-12-03T20:34:17,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741844_1020 (size=5170) 2024-12-03T20:34:17,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741844_1020 (size=5170) 2024-12-03T20:34:17,144 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/.tmp/a/a2460504cc55496d9c49cb33ba994307 2024-12-03T20:34:17,181 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/.tmp/a/a2460504cc55496d9c49cb33ba994307 as hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/a/a2460504cc55496d9c49cb33ba994307 2024-12-03T20:34:17,190 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/a/a2460504cc55496d9c49cb33ba994307, entries=2, sequenceid=2, filesize=5.0 K 2024-12-03T20:34:17,194 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 76530ff022f88fcb2ea00d9ba12621f7 in 106ms, sequenceid=2, compaction requested=false; wal=null 2024-12-03T20:34:17,196 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/recovered.edits/0000000000000000002 2024-12-03T20:34:17,197 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 76530ff022f88fcb2ea00d9ba12621f7 2024-12-03T20:34:17,197 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 76530ff022f88fcb2ea00d9ba12621f7 2024-12-03T20:34:17,200 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 76530ff022f88fcb2ea00d9ba12621f7 2024-12-03T20:34:17,203 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/76530ff022f88fcb2ea00d9ba12621f7/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-03T20:34:17,204 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 76530ff022f88fcb2ea00d9ba12621f7; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63757830, jitterRate=-0.04993429780006409}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T20:34:17,205 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 76530ff022f88fcb2ea00d9ba12621f7: Writing region info on filesystem at 1733258057066Initializing all the Stores at 1733258057068 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258057068Obtaining lock to block concurrent updates at 1733258057088 (+20 ms)Preparing flush snapshotting stores in 76530ff022f88fcb2ea00d9ba12621f7 at 1733258057088Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733258056822.76530ff022f88fcb2ea00d9ba12621f7., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733258057090 (+2 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733258056822.76530ff022f88fcb2ea00d9ba12621f7. at 1733258057090Flushing 76530ff022f88fcb2ea00d9ba12621f7/a: creating writer at 1733258057091 (+1 ms)Flushing 76530ff022f88fcb2ea00d9ba12621f7/a: appending metadata at 1733258057123 (+32 ms)Flushing 76530ff022f88fcb2ea00d9ba12621f7/a: closing flushed file at 1733258057125 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bfa95a5: reopening flushed file at 1733258057179 (+54 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 76530ff022f88fcb2ea00d9ba12621f7 in 106ms, sequenceid=2, compaction requested=false; wal=null at 1733258057195 (+16 ms)Cleaning up temporary data from old regions at 1733258057197 (+2 ms)Region opened successfully at 1733258057205 (+8 ms) 2024-12-03T20:34:17,230 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=382 (was 370) Potentially hanging thread: PacketResponder: BP-1199604065-172.17.0.3-1733258049083:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:39361/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:42508 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:40206 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:42004 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:42502 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1199604065-172.17.0.3-1733258049083:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:42132 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1199604065-172.17.0.3-1733258049083:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:40044 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=297 (was 297), ProcessCount=11 (was 11), AvailableMemoryMB=2407 (was 2428) 2024-12-03T20:34:17,239 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=382, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=297, ProcessCount=11, AvailableMemoryMB=2407 2024-12-03T20:34:17,252 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:17,254 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:17,255 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T20:34:17,259 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-43429783, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/hregion-43429783, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:17,270 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-43429783/hregion-43429783.1733258057260, exclude list is [], retry=0 2024-12-03T20:34:17,273 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:17,273 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:17,273 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:17,276 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-43429783/hregion-43429783.1733258057260 2024-12-03T20:34:17,276 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:36683:36683)] 2024-12-03T20:34:17,276 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 419cf3f4111dfd507910e9fde12a7be5, NAME => 'testReplayEditsWrittenIntoWAL,,1733258057253.419cf3f4111dfd507910e9fde12a7be5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39361/hbase 2024-12-03T20:34:17,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741846_1022 (size=64) 2024-12-03T20:34:17,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741846_1022 (size=64) 2024-12-03T20:34:17,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741846_1022 (size=64) 2024-12-03T20:34:17,287 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733258057253.419cf3f4111dfd507910e9fde12a7be5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:17,289 INFO [StoreOpener-419cf3f4111dfd507910e9fde12a7be5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 419cf3f4111dfd507910e9fde12a7be5 2024-12-03T20:34:17,291 INFO [StoreOpener-419cf3f4111dfd507910e9fde12a7be5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 419cf3f4111dfd507910e9fde12a7be5 columnFamilyName a 2024-12-03T20:34:17,291 DEBUG [StoreOpener-419cf3f4111dfd507910e9fde12a7be5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:17,292 INFO [StoreOpener-419cf3f4111dfd507910e9fde12a7be5-1 {}] regionserver.HStore(327): Store=419cf3f4111dfd507910e9fde12a7be5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:17,292 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 419cf3f4111dfd507910e9fde12a7be5 2024-12-03T20:34:17,293 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5 2024-12-03T20:34:17,294 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5 2024-12-03T20:34:17,294 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 419cf3f4111dfd507910e9fde12a7be5 2024-12-03T20:34:17,294 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 419cf3f4111dfd507910e9fde12a7be5 2024-12-03T20:34:17,297 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 419cf3f4111dfd507910e9fde12a7be5 2024-12-03T20:34:17,300 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T20:34:17,301 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 419cf3f4111dfd507910e9fde12a7be5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59317818, jitterRate=-0.116095632314682}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T20:34:17,301 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 419cf3f4111dfd507910e9fde12a7be5: Writing region info on filesystem at 1733258057287Initializing all the Stores at 1733258057289 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258057289Cleaning up temporary data from old regions at 1733258057294 (+5 ms)Region opened successfully at 1733258057301 (+7 ms) 2024-12-03T20:34:17,301 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 419cf3f4111dfd507910e9fde12a7be5, disabling compactions & flushes 2024-12-03T20:34:17,301 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733258057253.419cf3f4111dfd507910e9fde12a7be5. 2024-12-03T20:34:17,301 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733258057253.419cf3f4111dfd507910e9fde12a7be5. 2024-12-03T20:34:17,301 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733258057253.419cf3f4111dfd507910e9fde12a7be5. after waiting 0 ms 2024-12-03T20:34:17,301 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733258057253.419cf3f4111dfd507910e9fde12a7be5. 2024-12-03T20:34:17,302 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733258057253.419cf3f4111dfd507910e9fde12a7be5. 2024-12-03T20:34:17,302 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 419cf3f4111dfd507910e9fde12a7be5: Waiting for close lock at 1733258057301Disabling compacts and flushes for region at 1733258057301Disabling writes for close at 1733258057301Writing region close event to WAL at 1733258057302 (+1 ms)Closed at 1733258057302 2024-12-03T20:34:17,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741845_1021 (size=95) 2024-12-03T20:34:17,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741845_1021 (size=95) 2024-12-03T20:34:17,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741845_1021 (size=95) 2024-12-03T20:34:17,308 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T20:34:17,308 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-43429783:(num 1733258057260) 2024-12-03T20:34:17,309 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T20:34:17,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741847_1023 (size=320) 2024-12-03T20:34:17,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741847_1023 (size=320) 2024-12-03T20:34:17,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741847_1023 (size=320) 2024-12-03T20:34:17,325 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T20:34:17,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741848_1024 (size=253) 2024-12-03T20:34:17,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741848_1024 (size=253) 2024-12-03T20:34:17,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741848_1024 (size=253) 2024-12-03T20:34:17,350 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-2, size=253 (253bytes) 2024-12-03T20:34:17,350 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-2 2024-12-03T20:34:17,351 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-2 after 1ms 2024-12-03T20:34:17,354 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:17,355 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-2 took 5ms 2024-12-03T20:34:17,357 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-2 so closing down 2024-12-03T20:34:17,357 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T20:34:17,359 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-03T20:34:17,361 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000002-wal-2.temp 2024-12-03T20:34:17,361 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T20:34:17,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741849_1025 (size=253) 2024-12-03T20:34:17,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741849_1025 (size=253) 2024-12-03T20:34:17,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741849_1025 (size=253) 2024-12-03T20:34:17,370 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T20:34:17,372 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000002 2024-12-03T20:34:17,372 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 17 ms; skipped=0; WAL=hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-12-03T20:34:17,372 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-2, journal: Splitting hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-2, size=253 (253bytes) at 1733258057350Finishing writing output for hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-2 so closing down at 1733258057357 (+7 ms)Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000002-wal-2.temp at 1733258057361 (+4 ms)3 split writer threads finished at 1733258057361Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733258057370 (+9 ms)Rename recovered edits hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000002 at 1733258057372 (+2 ms)Processed 1 edits across 1 Regions in 17 ms; skipped=0; WAL=hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1733258057372 2024-12-03T20:34:17,388 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-1, size=320 (320bytes) 2024-12-03T20:34:17,388 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-1 2024-12-03T20:34:17,389 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-1 after 1ms 2024-12-03T20:34:17,392 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:17,392 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-1 took 4ms 2024-12-03T20:34:17,395 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-1 so closing down 2024-12-03T20:34:17,395 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T20:34:17,397 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-03T20:34:17,399 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000001-wal-1.temp 2024-12-03T20:34:17,399 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T20:34:17,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741850_1026 (size=320) 2024-12-03T20:34:17,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741850_1026 (size=320) 2024-12-03T20:34:17,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741850_1026 (size=320) 2024-12-03T20:34:17,407 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T20:34:17,411 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:17,413 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000002, length=253 2024-12-03T20:34:17,415 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000002 2024-12-03T20:34:17,415 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 22 ms; skipped=0; WAL=hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-12-03T20:34:17,415 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-1, journal: Splitting hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-1, size=320 (320bytes) at 1733258057388Finishing writing output for hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-1 so closing down at 1733258057395 (+7 ms)Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000001-wal-1.temp at 1733258057399 (+4 ms)3 split writer threads finished at 1733258057399Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733258057407 (+8 ms)Rename recovered edits hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000002 at 1733258057415 (+8 ms)Processed 2 edits across 1 Regions in 22 ms; skipped=0; WAL=hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1733258057415 2024-12-03T20:34:17,415 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T20:34:17,418 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:17,429 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal.1733258057419, exclude list is [], retry=0 2024-12-03T20:34:17,432 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:17,433 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:17,433 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:17,435 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal.1733258057419 2024-12-03T20:34:17,436 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:34005:34005)] 2024-12-03T20:34:17,436 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 419cf3f4111dfd507910e9fde12a7be5, NAME => 'testReplayEditsWrittenIntoWAL,,1733258057253.419cf3f4111dfd507910e9fde12a7be5.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:17,436 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733258057253.419cf3f4111dfd507910e9fde12a7be5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:17,436 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 419cf3f4111dfd507910e9fde12a7be5 2024-12-03T20:34:17,436 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 419cf3f4111dfd507910e9fde12a7be5 2024-12-03T20:34:17,438 INFO [StoreOpener-419cf3f4111dfd507910e9fde12a7be5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 419cf3f4111dfd507910e9fde12a7be5 2024-12-03T20:34:17,440 INFO [StoreOpener-419cf3f4111dfd507910e9fde12a7be5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 419cf3f4111dfd507910e9fde12a7be5 columnFamilyName a 2024-12-03T20:34:17,440 DEBUG [StoreOpener-419cf3f4111dfd507910e9fde12a7be5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:17,441 INFO [StoreOpener-419cf3f4111dfd507910e9fde12a7be5-1 {}] regionserver.HStore(327): Store=419cf3f4111dfd507910e9fde12a7be5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:17,441 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 419cf3f4111dfd507910e9fde12a7be5 2024-12-03T20:34:17,442 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5 2024-12-03T20:34:17,444 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5 2024-12-03T20:34:17,445 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000002 2024-12-03T20:34:17,448 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:17,450 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000002 2024-12-03T20:34:17,450 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 419cf3f4111dfd507910e9fde12a7be5 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-03T20:34:17,465 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/.tmp/a/24936b3b9ae2466497d5a5a32357b31a is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733258057308/Put/seqid=0 2024-12-03T20:34:17,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741852_1028 (size=5170) 2024-12-03T20:34:17,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741852_1028 (size=5170) 2024-12-03T20:34:17,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741852_1028 (size=5170) 2024-12-03T20:34:17,473 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/.tmp/a/24936b3b9ae2466497d5a5a32357b31a 2024-12-03T20:34:17,481 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/.tmp/a/24936b3b9ae2466497d5a5a32357b31a as hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/a/24936b3b9ae2466497d5a5a32357b31a 2024-12-03T20:34:17,489 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/a/24936b3b9ae2466497d5a5a32357b31a, entries=2, sequenceid=2, filesize=5.0 K 2024-12-03T20:34:17,489 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 419cf3f4111dfd507910e9fde12a7be5 in 39ms, sequenceid=2, compaction requested=false; wal=null 2024-12-03T20:34:17,490 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/0000000000000000002 2024-12-03T20:34:17,490 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 419cf3f4111dfd507910e9fde12a7be5 2024-12-03T20:34:17,490 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 419cf3f4111dfd507910e9fde12a7be5 2024-12-03T20:34:17,493 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 419cf3f4111dfd507910e9fde12a7be5 2024-12-03T20:34:17,496 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/419cf3f4111dfd507910e9fde12a7be5/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-03T20:34:17,497 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 419cf3f4111dfd507910e9fde12a7be5; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72741612, jitterRate=0.083934485912323}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T20:34:17,497 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 419cf3f4111dfd507910e9fde12a7be5: Writing region info on filesystem at 1733258057436Initializing all the Stores at 1733258057438 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258057438Obtaining lock to block concurrent updates at 1733258057450 (+12 ms)Preparing flush snapshotting stores in 419cf3f4111dfd507910e9fde12a7be5 at 1733258057450Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733258057253.419cf3f4111dfd507910e9fde12a7be5., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733258057450Flushing stores of testReplayEditsWrittenIntoWAL,,1733258057253.419cf3f4111dfd507910e9fde12a7be5. at 1733258057450Flushing 419cf3f4111dfd507910e9fde12a7be5/a: creating writer at 1733258057450Flushing 419cf3f4111dfd507910e9fde12a7be5/a: appending metadata at 1733258057464 (+14 ms)Flushing 419cf3f4111dfd507910e9fde12a7be5/a: closing flushed file at 1733258057464Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30a2736b: reopening flushed file at 1733258057480 (+16 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 419cf3f4111dfd507910e9fde12a7be5 in 39ms, sequenceid=2, compaction requested=false; wal=null at 1733258057489 (+9 ms)Cleaning up temporary data from old regions at 1733258057490 (+1 ms)Region opened successfully at 1733258057497 (+7 ms) 2024-12-03T20:34:17,518 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=392 (was 382) Potentially hanging thread: PacketResponder: BP-1199604065-172.17.0.3-1733258049083:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:40290 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:42004 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:42502 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1199604065-172.17.0.3-1733258049083:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:42584 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:40044 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1199604065-172.17.0.3-1733258049083:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:42206 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=297 (was 297), ProcessCount=11 (was 11), AvailableMemoryMB=2397 (was 2407) 2024-12-03T20:34:17,530 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=392, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=297, ProcessCount=11, AvailableMemoryMB=2396 2024-12-03T20:34:17,543 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:17,545 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:17,546 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T20:34:17,549 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-79006119, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/hregion-79006119, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:17,561 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-79006119/hregion-79006119.1733258057549, exclude list is [], retry=0 2024-12-03T20:34:17,564 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:17,565 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:17,565 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:17,567 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-79006119/hregion-79006119.1733258057549 2024-12-03T20:34:17,568 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:33459:33459)] 2024-12-03T20:34:17,568 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 65f8b6e22d9834b5557488ef1fca5ac2, NAME => 'testReplayEditsWrittenIntoWAL,,1733258057544.65f8b6e22d9834b5557488ef1fca5ac2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39361/hbase 2024-12-03T20:34:17,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741854_1030 (size=64) 2024-12-03T20:34:17,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741854_1030 (size=64) 2024-12-03T20:34:17,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741854_1030 (size=64) 2024-12-03T20:34:17,578 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733258057544.65f8b6e22d9834b5557488ef1fca5ac2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:17,580 INFO [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 65f8b6e22d9834b5557488ef1fca5ac2 2024-12-03T20:34:17,581 INFO [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65f8b6e22d9834b5557488ef1fca5ac2 columnFamilyName a 2024-12-03T20:34:17,582 DEBUG [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:17,582 INFO [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] regionserver.HStore(327): Store=65f8b6e22d9834b5557488ef1fca5ac2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:17,582 INFO [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 65f8b6e22d9834b5557488ef1fca5ac2 2024-12-03T20:34:17,583 INFO [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65f8b6e22d9834b5557488ef1fca5ac2 columnFamilyName b 2024-12-03T20:34:17,584 DEBUG [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:17,584 INFO [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] regionserver.HStore(327): Store=65f8b6e22d9834b5557488ef1fca5ac2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:17,584 INFO [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 65f8b6e22d9834b5557488ef1fca5ac2 2024-12-03T20:34:17,586 INFO [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65f8b6e22d9834b5557488ef1fca5ac2 columnFamilyName c 2024-12-03T20:34:17,586 DEBUG [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:17,586 INFO [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] regionserver.HStore(327): Store=65f8b6e22d9834b5557488ef1fca5ac2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:17,586 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 65f8b6e22d9834b5557488ef1fca5ac2 2024-12-03T20:34:17,587 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2 2024-12-03T20:34:17,587 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2 2024-12-03T20:34:17,589 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 65f8b6e22d9834b5557488ef1fca5ac2 2024-12-03T20:34:17,589 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 65f8b6e22d9834b5557488ef1fca5ac2 2024-12-03T20:34:17,589 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T20:34:17,591 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 65f8b6e22d9834b5557488ef1fca5ac2 2024-12-03T20:34:17,593 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T20:34:17,593 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 65f8b6e22d9834b5557488ef1fca5ac2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67980836, jitterRate=0.012993395328521729}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T20:34:17,594 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 65f8b6e22d9834b5557488ef1fca5ac2: Writing region info on filesystem at 1733258057578Initializing all the Stores at 1733258057579 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258057580 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258057580Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258057580Cleaning up temporary data from old regions at 1733258057589 (+9 ms)Region opened successfully at 1733258057594 (+5 ms) 2024-12-03T20:34:17,594 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 65f8b6e22d9834b5557488ef1fca5ac2, disabling compactions & flushes 2024-12-03T20:34:17,594 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733258057544.65f8b6e22d9834b5557488ef1fca5ac2. 2024-12-03T20:34:17,594 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733258057544.65f8b6e22d9834b5557488ef1fca5ac2. 2024-12-03T20:34:17,594 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733258057544.65f8b6e22d9834b5557488ef1fca5ac2. after waiting 0 ms 2024-12-03T20:34:17,594 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733258057544.65f8b6e22d9834b5557488ef1fca5ac2. 2024-12-03T20:34:17,594 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733258057544.65f8b6e22d9834b5557488ef1fca5ac2. 2024-12-03T20:34:17,595 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 65f8b6e22d9834b5557488ef1fca5ac2: Waiting for close lock at 1733258057594Disabling compacts and flushes for region at 1733258057594Disabling writes for close at 1733258057594Writing region close event to WAL at 1733258057594Closed at 1733258057594 2024-12-03T20:34:17,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741853_1029 (size=95) 2024-12-03T20:34:17,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741853_1029 (size=95) 2024-12-03T20:34:17,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741853_1029 (size=95) 2024-12-03T20:34:17,601 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T20:34:17,601 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-79006119:(num 1733258057549) 2024-12-03T20:34:17,601 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T20:34:17,604 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:17,615 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258057604, exclude list is [], retry=0 2024-12-03T20:34:17,618 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:17,619 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:17,619 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:17,621 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258057604 2024-12-03T20:34:17,621 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:36683:36683)] 2024-12-03T20:34:17,814 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258057604, size=0 (0bytes) 2024-12-03T20:34:17,814 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258057604 might be still open, length is 0 2024-12-03T20:34:17,814 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258057604 2024-12-03T20:34:17,815 WARN [IPC Server handler 2 on default port 39361 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258057604 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-03T20:34:17,816 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258057604 after 1ms 2024-12-03T20:34:18,567 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:42236 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36623:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42236 dst: /127.0.0.1:36623 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36623 remote=/127.0.0.1:42236]. Total timeout mills is 60000, 59219 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:18,568 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:40322 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:46119:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40322 dst: /127.0.0.1:46119 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:18,568 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:42616 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33321:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42616 dst: /127.0.0.1:33321 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:18,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741855_1032 (size=263633) 2024-12-03T20:34:18,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741855_1032 (size=263633) 2024-12-03T20:34:18,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741855_1032 (size=263633) 2024-12-03T20:34:21,817 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258057604 after 4003ms 2024-12-03T20:34:21,826 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258057604: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:21,828 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258057604 took 4014ms 2024-12-03T20:34:21,833 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733258057604.temp 2024-12-03T20:34:21,837 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/recovered.edits/0000000000000000001-wal.1733258057604.temp 2024-12-03T20:34:21,955 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258057604; continuing. 2024-12-03T20:34:21,955 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258057604 so closing down 2024-12-03T20:34:21,956 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T20:34:21,956 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T20:34:21,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741856_1033 (size=263641) 2024-12-03T20:34:21,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741856_1033 (size=263641) 2024-12-03T20:34:21,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741856_1033 (size=263641) 2024-12-03T20:34:21,960 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/recovered.edits/0000000000000000001-wal.1733258057604.temp (wrote 3002 edits, skipped 0 edits in 66 ms) 2024-12-03T20:34:21,963 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/recovered.edits/0000000000000000001-wal.1733258057604.temp to hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/recovered.edits/0000000000000003002 2024-12-03T20:34:21,963 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 135 ms; skipped=0; WAL=hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258057604, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T20:34:21,963 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258057604, journal: Splitting hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258057604, size=0 (0bytes) at 1733258057814Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/recovered.edits/0000000000000000001-wal.1733258057604.temp at 1733258061838 (+4024 ms)Split 1024 edits, skipped 0 edits. at 1733258061901 (+63 ms)Split 2048 edits, skipped 0 edits. at 1733258061930 (+29 ms)Finishing writing output for hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258057604 so closing down at 1733258061955 (+25 ms)3 split writer threads finished at 1733258061956 (+1 ms)Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/recovered.edits/0000000000000000001-wal.1733258057604.temp (wrote 3002 edits, skipped 0 edits in 66 ms) at 1733258061960 (+4 ms)Rename recovered edits hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/recovered.edits/0000000000000000001-wal.1733258057604.temp to hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/recovered.edits/0000000000000003002 at 1733258061963 (+3 ms)Processed 3002 edits across 1 Regions in 135 ms; skipped=0; WAL=hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258057604, size=0, length=0, corrupted=false, cancelled=false at 1733258061963 2024-12-03T20:34:21,965 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258057604 to hdfs://localhost:39361/hbase/oldWALs/wal.1733258057604 2024-12-03T20:34:21,967 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/recovered.edits/0000000000000003002 2024-12-03T20:34:21,967 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T20:34:21,969 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T20:34:21,970 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:21,988 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258061970, exclude list is [], retry=0 2024-12-03T20:34:21,992 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:21,992 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:21,993 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:21,997 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258061970 2024-12-03T20:34:21,997 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:34005:34005)] 2024-12-03T20:34:21,998 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733258057544.65f8b6e22d9834b5557488ef1fca5ac2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:22,000 INFO [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 65f8b6e22d9834b5557488ef1fca5ac2 2024-12-03T20:34:22,001 INFO [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65f8b6e22d9834b5557488ef1fca5ac2 columnFamilyName a 2024-12-03T20:34:22,001 DEBUG [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:22,002 INFO [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] regionserver.HStore(327): Store=65f8b6e22d9834b5557488ef1fca5ac2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:22,002 INFO [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 65f8b6e22d9834b5557488ef1fca5ac2 2024-12-03T20:34:22,003 INFO [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65f8b6e22d9834b5557488ef1fca5ac2 columnFamilyName b 2024-12-03T20:34:22,003 DEBUG [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:22,004 INFO [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] regionserver.HStore(327): Store=65f8b6e22d9834b5557488ef1fca5ac2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:22,004 INFO [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 65f8b6e22d9834b5557488ef1fca5ac2 2024-12-03T20:34:22,005 INFO [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65f8b6e22d9834b5557488ef1fca5ac2 columnFamilyName c 2024-12-03T20:34:22,005 DEBUG [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:22,006 INFO [StoreOpener-65f8b6e22d9834b5557488ef1fca5ac2-1 {}] regionserver.HStore(327): Store=65f8b6e22d9834b5557488ef1fca5ac2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:22,006 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 65f8b6e22d9834b5557488ef1fca5ac2 2024-12-03T20:34:22,007 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2 2024-12-03T20:34:22,009 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2 2024-12-03T20:34:22,010 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/recovered.edits/0000000000000003002 2024-12-03T20:34:22,013 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:22,030 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T20:34:22,051 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T20:34:22,372 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 65f8b6e22d9834b5557488ef1fca5ac2 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T20:34:22,406 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/.tmp/a/3db70408a83044859fea1c61dba84350 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733258057628/Put/seqid=0 2024-12-03T20:34:22,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741858_1035 (size=50463) 2024-12-03T20:34:22,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741858_1035 (size=50463) 2024-12-03T20:34:22,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741858_1035 (size=50463) 2024-12-03T20:34:22,415 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/.tmp/a/3db70408a83044859fea1c61dba84350 2024-12-03T20:34:22,422 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/.tmp/a/3db70408a83044859fea1c61dba84350 as hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/a/3db70408a83044859fea1c61dba84350 2024-12-03T20:34:22,429 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/a/3db70408a83044859fea1c61dba84350, entries=754, sequenceid=754, filesize=49.3 K 2024-12-03T20:34:22,429 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 65f8b6e22d9834b5557488ef1fca5ac2 in 57ms, sequenceid=754, compaction requested=false; wal=null 2024-12-03T20:34:22,465 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T20:34:22,466 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 65f8b6e22d9834b5557488ef1fca5ac2 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T20:34:22,474 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/.tmp/a/3e09f0f39c6d4dfb80961490e4f67bdc is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733258057663/Put/seqid=0 2024-12-03T20:34:22,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741859_1036 (size=20072) 2024-12-03T20:34:22,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741859_1036 (size=20072) 2024-12-03T20:34:22,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741859_1036 (size=20072) 2024-12-03T20:34:22,887 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/.tmp/a/3e09f0f39c6d4dfb80961490e4f67bdc 2024-12-03T20:34:22,916 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/.tmp/b/870474ac554d4a09a84e5bc86a3f132e is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733258057683/Put/seqid=0 2024-12-03T20:34:22,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741860_1037 (size=35835) 2024-12-03T20:34:22,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741860_1037 (size=35835) 2024-12-03T20:34:22,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741860_1037 (size=35835) 2024-12-03T20:34:22,925 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/.tmp/b/870474ac554d4a09a84e5bc86a3f132e 2024-12-03T20:34:22,932 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/.tmp/a/3e09f0f39c6d4dfb80961490e4f67bdc as hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/a/3e09f0f39c6d4dfb80961490e4f67bdc 2024-12-03T20:34:22,939 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/a/3e09f0f39c6d4dfb80961490e4f67bdc, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-03T20:34:22,941 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/.tmp/b/870474ac554d4a09a84e5bc86a3f132e as hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/b/870474ac554d4a09a84e5bc86a3f132e 2024-12-03T20:34:22,948 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/b/870474ac554d4a09a84e5bc86a3f132e, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-03T20:34:22,949 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 65f8b6e22d9834b5557488ef1fca5ac2 in 482ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-03T20:34:22,966 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T20:34:22,967 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 65f8b6e22d9834b5557488ef1fca5ac2 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T20:34:22,974 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/.tmp/b/ea240725dd0f4ef8b927068dad8f9499 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733258057713/Put/seqid=0 2024-12-03T20:34:22,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741861_1038 (size=35082) 2024-12-03T20:34:22,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741861_1038 (size=35082) 2024-12-03T20:34:22,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741861_1038 (size=35082) 2024-12-03T20:34:22,983 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/.tmp/b/ea240725dd0f4ef8b927068dad8f9499 2024-12-03T20:34:23,004 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/.tmp/c/55b014b590a044988b6553224540cfa5 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733258057751/Put/seqid=0 2024-12-03T20:34:23,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741862_1039 (size=20825) 2024-12-03T20:34:23,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741862_1039 (size=20825) 2024-12-03T20:34:23,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741862_1039 (size=20825) 2024-12-03T20:34:23,013 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/.tmp/c/55b014b590a044988b6553224540cfa5 2024-12-03T20:34:23,021 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/.tmp/b/ea240725dd0f4ef8b927068dad8f9499 as hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/b/ea240725dd0f4ef8b927068dad8f9499 2024-12-03T20:34:23,027 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/b/ea240725dd0f4ef8b927068dad8f9499, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-03T20:34:23,029 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/.tmp/c/55b014b590a044988b6553224540cfa5 as hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/c/55b014b590a044988b6553224540cfa5 2024-12-03T20:34:23,034 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/c/55b014b590a044988b6553224540cfa5, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-03T20:34:23,035 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 65f8b6e22d9834b5557488ef1fca5ac2 in 68ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-03T20:34:23,052 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733258057785/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733258057544.65f8b6e22d9834b5557488ef1fca5ac2. 2024-12-03T20:34:23,056 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/recovered.edits/0000000000000003002 2024-12-03T20:34:23,056 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T20:34:23,056 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 65f8b6e22d9834b5557488ef1fca5ac2 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-03T20:34:23,065 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/.tmp/c/95337146e9d547f989c33f993bdb6bf6 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733258057757/Put/seqid=0 2024-12-03T20:34:23,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741863_1040 (size=50301) 2024-12-03T20:34:23,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741863_1040 (size=50301) 2024-12-03T20:34:23,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741863_1040 (size=50301) 2024-12-03T20:34:23,073 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/.tmp/c/95337146e9d547f989c33f993bdb6bf6 2024-12-03T20:34:23,080 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 95337146e9d547f989c33f993bdb6bf6 2024-12-03T20:34:23,082 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/.tmp/c/95337146e9d547f989c33f993bdb6bf6 as hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/c/95337146e9d547f989c33f993bdb6bf6 2024-12-03T20:34:23,087 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 95337146e9d547f989c33f993bdb6bf6 2024-12-03T20:34:23,088 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/c/95337146e9d547f989c33f993bdb6bf6, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-03T20:34:23,088 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 65f8b6e22d9834b5557488ef1fca5ac2 in 32ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-03T20:34:23,089 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/recovered.edits/0000000000000003002 2024-12-03T20:34:23,090 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 65f8b6e22d9834b5557488ef1fca5ac2 2024-12-03T20:34:23,091 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 65f8b6e22d9834b5557488ef1fca5ac2 2024-12-03T20:34:23,091 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T20:34:23,094 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 65f8b6e22d9834b5557488ef1fca5ac2 2024-12-03T20:34:23,098 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenIntoWAL/65f8b6e22d9834b5557488ef1fca5ac2/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-03T20:34:23,100 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 65f8b6e22d9834b5557488ef1fca5ac2; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69986023, jitterRate=0.04287301003932953}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T20:34:23,100 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 65f8b6e22d9834b5557488ef1fca5ac2: Writing region info on filesystem at 1733258061998Initializing all the Stores at 1733258061999 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258061999Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258061999Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258061999Cleaning up temporary data from old regions at 1733258063091 (+1092 ms)Region opened successfully at 1733258063100 (+9 ms) 2024-12-03T20:34:23,154 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 65f8b6e22d9834b5557488ef1fca5ac2, disabling compactions & flushes 2024-12-03T20:34:23,154 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733258057544.65f8b6e22d9834b5557488ef1fca5ac2. 2024-12-03T20:34:23,154 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733258057544.65f8b6e22d9834b5557488ef1fca5ac2. 2024-12-03T20:34:23,154 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733258057544.65f8b6e22d9834b5557488ef1fca5ac2. after waiting 0 ms 2024-12-03T20:34:23,154 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733258057544.65f8b6e22d9834b5557488ef1fca5ac2. 2024-12-03T20:34:23,156 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733258057544.65f8b6e22d9834b5557488ef1fca5ac2. 2024-12-03T20:34:23,156 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 65f8b6e22d9834b5557488ef1fca5ac2: Waiting for close lock at 1733258063154Disabling compacts and flushes for region at 1733258063154Disabling writes for close at 1733258063154Writing region close event to WAL at 1733258063156 (+2 ms)Closed at 1733258063156 2024-12-03T20:34:23,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741857_1034 (size=95) 2024-12-03T20:34:23,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741857_1034 (size=95) 2024-12-03T20:34:23,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741857_1034 (size=95) 2024-12-03T20:34:23,164 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T20:34:23,164 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733258061970) 2024-12-03T20:34:23,176 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=406 (was 392) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42295 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:35521 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1198915196_22 at /127.0.0.1:42256 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:39361 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1198915196_22 at /127.0.0.1:55870 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:42295 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1198915196_22 at /127.0.0.1:48512 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:39361 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1198915196_22 at /127.0.0.1:55844 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35521 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=273 (was 297), ProcessCount=11 (was 11), AvailableMemoryMB=2333 (was 2396) 2024-12-03T20:34:23,187 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=406, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=273, ProcessCount=11, AvailableMemoryMB=2333 2024-12-03T20:34:23,203 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:23,205 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:23,206 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T20:34:23,209 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-44175798, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/hregion-44175798, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:23,222 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-44175798/hregion-44175798.1733258063210, exclude list is [], retry=0 2024-12-03T20:34:23,225 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:23,225 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:23,226 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:23,228 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-44175798/hregion-44175798.1733258063210 2024-12-03T20:34:23,231 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:36683:36683)] 2024-12-03T20:34:23,231 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8e57dc10d2fffd727c780406ed8f9136, NAME => 'test2727,,1733258063204.8e57dc10d2fffd727c780406ed8f9136.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39361/hbase 2024-12-03T20:34:23,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741865_1042 (size=43) 2024-12-03T20:34:23,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741865_1042 (size=43) 2024-12-03T20:34:23,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741865_1042 (size=43) 2024-12-03T20:34:23,244 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733258063204.8e57dc10d2fffd727c780406ed8f9136.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:23,249 INFO [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8e57dc10d2fffd727c780406ed8f9136 2024-12-03T20:34:23,252 INFO [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e57dc10d2fffd727c780406ed8f9136 columnFamilyName a 2024-12-03T20:34:23,252 DEBUG [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:23,253 INFO [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] regionserver.HStore(327): Store=8e57dc10d2fffd727c780406ed8f9136/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:23,253 INFO [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8e57dc10d2fffd727c780406ed8f9136 2024-12-03T20:34:23,255 INFO [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e57dc10d2fffd727c780406ed8f9136 columnFamilyName b 2024-12-03T20:34:23,255 DEBUG [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:23,255 INFO [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] regionserver.HStore(327): Store=8e57dc10d2fffd727c780406ed8f9136/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:23,256 INFO [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8e57dc10d2fffd727c780406ed8f9136 2024-12-03T20:34:23,257 INFO [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e57dc10d2fffd727c780406ed8f9136 columnFamilyName c 2024-12-03T20:34:23,257 DEBUG [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:23,258 INFO [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] regionserver.HStore(327): Store=8e57dc10d2fffd727c780406ed8f9136/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:23,258 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8e57dc10d2fffd727c780406ed8f9136 2024-12-03T20:34:23,259 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136 2024-12-03T20:34:23,259 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136 2024-12-03T20:34:23,260 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8e57dc10d2fffd727c780406ed8f9136 2024-12-03T20:34:23,261 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8e57dc10d2fffd727c780406ed8f9136 2024-12-03T20:34:23,261 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T20:34:23,262 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8e57dc10d2fffd727c780406ed8f9136 2024-12-03T20:34:23,265 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T20:34:23,266 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8e57dc10d2fffd727c780406ed8f9136; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73309572, jitterRate=0.09239774942398071}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T20:34:23,267 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8e57dc10d2fffd727c780406ed8f9136: Writing region info on filesystem at 1733258063244Initializing all the Stores at 1733258063245 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258063245Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258063249 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258063249Cleaning up temporary data from old regions at 1733258063261 (+12 ms)Region opened successfully at 1733258063267 (+6 ms) 2024-12-03T20:34:23,268 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8e57dc10d2fffd727c780406ed8f9136, disabling compactions & flushes 2024-12-03T20:34:23,268 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733258063204.8e57dc10d2fffd727c780406ed8f9136. 2024-12-03T20:34:23,268 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733258063204.8e57dc10d2fffd727c780406ed8f9136. 2024-12-03T20:34:23,268 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733258063204.8e57dc10d2fffd727c780406ed8f9136. after waiting 0 ms 2024-12-03T20:34:23,268 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733258063204.8e57dc10d2fffd727c780406ed8f9136. 2024-12-03T20:34:23,268 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733258063204.8e57dc10d2fffd727c780406ed8f9136. 2024-12-03T20:34:23,268 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8e57dc10d2fffd727c780406ed8f9136: Waiting for close lock at 1733258063267Disabling compacts and flushes for region at 1733258063267Disabling writes for close at 1733258063268 (+1 ms)Writing region close event to WAL at 1733258063268Closed at 1733258063268 2024-12-03T20:34:23,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741864_1041 (size=95) 2024-12-03T20:34:23,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741864_1041 (size=95) 2024-12-03T20:34:23,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741864_1041 (size=95) 2024-12-03T20:34:23,274 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T20:34:23,275 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-44175798:(num 1733258063210) 2024-12-03T20:34:23,275 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T20:34:23,277 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:23,289 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063278, exclude list is [], retry=0 2024-12-03T20:34:23,292 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:23,293 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:23,293 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:23,296 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063278 2024-12-03T20:34:23,296 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:36683:36683)] 2024-12-03T20:34:23,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741866_1043 (size=263359) 2024-12-03T20:34:23,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741866_1043 (size=263359) 2024-12-03T20:34:23,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741866_1043 (size=263359) 2024-12-03T20:34:23,441 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063278, size=257.2 K (263359bytes) 2024-12-03T20:34:23,441 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063278 2024-12-03T20:34:23,441 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063278 after 0ms 2024-12-03T20:34:23,444 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063278: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:23,446 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063278 took 4ms 2024-12-03T20:34:23,450 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733258063278.temp 2024-12-03T20:34:23,452 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000000001-wal.1733258063278.temp 2024-12-03T20:34:23,497 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063278 so closing down 2024-12-03T20:34:23,497 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T20:34:23,497 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T20:34:23,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741867_1044 (size=263359) 2024-12-03T20:34:23,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741867_1044 (size=263359) 2024-12-03T20:34:23,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741867_1044 (size=263359) 2024-12-03T20:34:23,501 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000000001-wal.1733258063278.temp (wrote 3000 edits, skipped 0 edits in 41 ms) 2024-12-03T20:34:23,503 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000000001-wal.1733258063278.temp to hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000003000 2024-12-03T20:34:23,503 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 57 ms; skipped=0; WAL=hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063278, size=257.2 K, length=263359, corrupted=false, cancelled=false 2024-12-03T20:34:23,503 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063278, journal: Splitting hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063278, size=257.2 K (263359bytes) at 1733258063441Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000000001-wal.1733258063278.temp at 1733258063452 (+11 ms)Split 1024 edits, skipped 0 edits. at 1733258063465 (+13 ms)Split 2048 edits, skipped 0 edits. at 1733258063481 (+16 ms)Finishing writing output for hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063278 so closing down at 1733258063497 (+16 ms)3 split writer threads finished at 1733258063497Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000000001-wal.1733258063278.temp (wrote 3000 edits, skipped 0 edits in 41 ms) at 1733258063501 (+4 ms)Rename recovered edits hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000000001-wal.1733258063278.temp to hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000003000 at 1733258063503 (+2 ms)Processed 3000 edits across 1 Regions in 57 ms; skipped=0; WAL=hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063278, size=257.2 K, length=263359, corrupted=false, cancelled=false at 1733258063503 2024-12-03T20:34:23,506 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063278 to hdfs://localhost:39361/hbase/oldWALs/wal.1733258063278 2024-12-03T20:34:23,507 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000003000 2024-12-03T20:34:23,508 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T20:34:23,510 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:23,524 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063511, exclude list is [], retry=0 2024-12-03T20:34:23,527 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:23,528 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:23,528 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:23,530 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063511 2024-12-03T20:34:23,531 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:36683:36683)] 2024-12-03T20:34:23,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741868_1045 (size=263486) 2024-12-03T20:34:23,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741868_1045 (size=263486) 2024-12-03T20:34:23,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741868_1045 (size=263486) 2024-12-03T20:34:23,726 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063511, size=257.3 K (263486bytes) 2024-12-03T20:34:23,726 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063511 2024-12-03T20:34:23,726 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063511 after 0ms 2024-12-03T20:34:23,729 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063511: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:23,730 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063511 took 5ms 2024-12-03T20:34:23,734 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733258063511.temp 2024-12-03T20:34:23,736 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000003001-wal.1733258063511.temp 2024-12-03T20:34:23,787 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T20:34:23,788 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T20:34:23,790 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T20:34:23,790 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T20:34:23,791 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T20:34:23,791 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T20:34:23,792 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-03T20:34:23,792 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-03T20:34:23,793 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063511 so closing down 2024-12-03T20:34:23,794 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T20:34:23,796 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T20:34:23,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741869_1046 (size=263486) 2024-12-03T20:34:23,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741869_1046 (size=263486) 2024-12-03T20:34:23,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741869_1046 (size=263486) 2024-12-03T20:34:23,800 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000003001-wal.1733258063511.temp (wrote 3000 edits, skipped 0 edits in 37 ms) 2024-12-03T20:34:23,802 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000003001-wal.1733258063511.temp to hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000006000 2024-12-03T20:34:23,802 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063511, size=257.3 K, length=263486, corrupted=false, cancelled=false 2024-12-03T20:34:23,803 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063511, journal: Splitting hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063511, size=257.3 K (263486bytes) at 1733258063726Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000003001-wal.1733258063511.temp at 1733258063736 (+10 ms)Split 1024 edits, skipped 0 edits. at 1733258063753 (+17 ms)Split 2048 edits, skipped 0 edits. at 1733258063774 (+21 ms)Finishing writing output for hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063511 so closing down at 1733258063793 (+19 ms)3 split writer threads finished at 1733258063796 (+3 ms)Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000003001-wal.1733258063511.temp (wrote 3000 edits, skipped 0 edits in 37 ms) at 1733258063800 (+4 ms)Rename recovered edits hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000003001-wal.1733258063511.temp to hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000006000 at 1733258063802 (+2 ms)Processed 3000 edits across 1 Regions in 71 ms; skipped=0; WAL=hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063511, size=257.3 K, length=263486, corrupted=false, cancelled=false at 1733258063802 2024-12-03T20:34:23,805 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063511 to hdfs://localhost:39361/hbase/oldWALs/wal.1733258063511 2024-12-03T20:34:23,806 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000006000 2024-12-03T20:34:23,806 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T20:34:23,808 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/test2727-manual,16010,1733258063202, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:23,823 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063809, exclude list is [], retry=0 2024-12-03T20:34:23,826 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:23,826 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:23,826 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:23,829 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733258063202/wal.1733258063809 2024-12-03T20:34:23,829 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:34005:34005)] 2024-12-03T20:34:23,829 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e57dc10d2fffd727c780406ed8f9136, NAME => 'test2727,,1733258063204.8e57dc10d2fffd727c780406ed8f9136.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:23,830 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733258063204.8e57dc10d2fffd727c780406ed8f9136.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:23,830 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8e57dc10d2fffd727c780406ed8f9136 2024-12-03T20:34:23,830 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8e57dc10d2fffd727c780406ed8f9136 2024-12-03T20:34:23,832 INFO [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8e57dc10d2fffd727c780406ed8f9136 2024-12-03T20:34:23,834 INFO [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e57dc10d2fffd727c780406ed8f9136 columnFamilyName a 2024-12-03T20:34:23,834 DEBUG [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:23,835 INFO [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] regionserver.HStore(327): Store=8e57dc10d2fffd727c780406ed8f9136/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:23,835 INFO [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8e57dc10d2fffd727c780406ed8f9136 2024-12-03T20:34:23,836 INFO [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e57dc10d2fffd727c780406ed8f9136 columnFamilyName b 2024-12-03T20:34:23,836 DEBUG [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:23,837 INFO [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] regionserver.HStore(327): Store=8e57dc10d2fffd727c780406ed8f9136/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:23,837 INFO [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8e57dc10d2fffd727c780406ed8f9136 2024-12-03T20:34:23,839 INFO [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e57dc10d2fffd727c780406ed8f9136 columnFamilyName c 2024-12-03T20:34:23,839 DEBUG [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:23,839 INFO [StoreOpener-8e57dc10d2fffd727c780406ed8f9136-1 {}] regionserver.HStore(327): Store=8e57dc10d2fffd727c780406ed8f9136/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:23,840 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8e57dc10d2fffd727c780406ed8f9136 2024-12-03T20:34:23,841 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136 2024-12-03T20:34:23,843 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136 2024-12-03T20:34:23,844 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000003000 2024-12-03T20:34:23,847 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:23,904 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000003000 2024-12-03T20:34:23,905 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000006000 2024-12-03T20:34:23,908 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:23,962 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000006000 2024-12-03T20:34:23,962 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8e57dc10d2fffd727c780406ed8f9136 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-03T20:34:23,983 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/.tmp/a/d8d59b7df3f9496c8b46b0162706c531 is 41, key is test2727/a:100/1733258063534/Put/seqid=0 2024-12-03T20:34:23,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741871_1048 (size=84227) 2024-12-03T20:34:23,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741871_1048 (size=84227) 2024-12-03T20:34:23,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741871_1048 (size=84227) 2024-12-03T20:34:23,991 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/.tmp/a/d8d59b7df3f9496c8b46b0162706c531 2024-12-03T20:34:24,025 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/.tmp/b/e91c8b907c604d21b56895143b9afad6 is 41, key is test2727/b:100/1733258063579/Put/seqid=0 2024-12-03T20:34:24,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741872_1049 (size=84609) 2024-12-03T20:34:24,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741872_1049 (size=84609) 2024-12-03T20:34:24,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741872_1049 (size=84609) 2024-12-03T20:34:24,030 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/.tmp/b/e91c8b907c604d21b56895143b9afad6 2024-12-03T20:34:24,054 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/.tmp/c/d967c8c4afa146438f9638eca07de35c is 41, key is test2727/c:100/1733258063651/Put/seqid=0 2024-12-03T20:34:24,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741873_1050 (size=84609) 2024-12-03T20:34:24,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741873_1050 (size=84609) 2024-12-03T20:34:24,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741873_1050 (size=84609) 2024-12-03T20:34:24,061 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/.tmp/c/d967c8c4afa146438f9638eca07de35c 2024-12-03T20:34:24,069 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/.tmp/a/d8d59b7df3f9496c8b46b0162706c531 as hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/a/d8d59b7df3f9496c8b46b0162706c531 2024-12-03T20:34:24,076 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/a/d8d59b7df3f9496c8b46b0162706c531, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-03T20:34:24,078 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/.tmp/b/e91c8b907c604d21b56895143b9afad6 as hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/b/e91c8b907c604d21b56895143b9afad6 2024-12-03T20:34:24,085 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/b/e91c8b907c604d21b56895143b9afad6, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-03T20:34:24,087 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/.tmp/c/d967c8c4afa146438f9638eca07de35c as hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/c/d967c8c4afa146438f9638eca07de35c 2024-12-03T20:34:24,094 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/c/d967c8c4afa146438f9638eca07de35c, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-03T20:34:24,094 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 8e57dc10d2fffd727c780406ed8f9136 in 132ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-03T20:34:24,095 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000003000 2024-12-03T20:34:24,096 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/0000000000000006000 2024-12-03T20:34:24,097 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8e57dc10d2fffd727c780406ed8f9136 2024-12-03T20:34:24,097 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8e57dc10d2fffd727c780406ed8f9136 2024-12-03T20:34:24,098 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T20:34:24,100 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8e57dc10d2fffd727c780406ed8f9136 2024-12-03T20:34:24,103 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/test2727/8e57dc10d2fffd727c780406ed8f9136/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-03T20:34:24,104 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8e57dc10d2fffd727c780406ed8f9136; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69902500, jitterRate=0.04162842035293579}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T20:34:24,106 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8e57dc10d2fffd727c780406ed8f9136: Writing region info on filesystem at 1733258063830Initializing all the Stores at 1733258063832 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258063832Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258063832Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258063832Obtaining lock to block concurrent updates at 1733258063962 (+130 ms)Preparing flush snapshotting stores in 8e57dc10d2fffd727c780406ed8f9136 at 1733258063962Finished memstore snapshotting test2727,,1733258063204.8e57dc10d2fffd727c780406ed8f9136., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733258063962Flushing stores of test2727,,1733258063204.8e57dc10d2fffd727c780406ed8f9136. at 1733258063962Flushing 8e57dc10d2fffd727c780406ed8f9136/a: creating writer at 1733258063962Flushing 8e57dc10d2fffd727c780406ed8f9136/a: appending metadata at 1733258063982 (+20 ms)Flushing 8e57dc10d2fffd727c780406ed8f9136/a: closing flushed file at 1733258063982Flushing 8e57dc10d2fffd727c780406ed8f9136/b: creating writer at 1733258064000 (+18 ms)Flushing 8e57dc10d2fffd727c780406ed8f9136/b: appending metadata at 1733258064023 (+23 ms)Flushing 8e57dc10d2fffd727c780406ed8f9136/b: closing flushed file at 1733258064023Flushing 8e57dc10d2fffd727c780406ed8f9136/c: creating writer at 1733258064036 (+13 ms)Flushing 8e57dc10d2fffd727c780406ed8f9136/c: appending metadata at 1733258064053 (+17 ms)Flushing 8e57dc10d2fffd727c780406ed8f9136/c: closing flushed file at 1733258064054 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5613c7f0: reopening flushed file at 1733258064068 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@379ff94c: reopening flushed file at 1733258064077 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@208a9c49: reopening flushed file at 1733258064085 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 8e57dc10d2fffd727c780406ed8f9136 in 132ms, sequenceid=6000, compaction requested=false; wal=null at 1733258064094 (+9 ms)Cleaning up temporary data from old regions at 1733258064097 (+3 ms)Region opened successfully at 1733258064106 (+9 ms) 2024-12-03T20:34:24,107 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-03T20:34:24,107 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8e57dc10d2fffd727c780406ed8f9136, disabling compactions & flushes 2024-12-03T20:34:24,107 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733258063204.8e57dc10d2fffd727c780406ed8f9136. 2024-12-03T20:34:24,107 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733258063204.8e57dc10d2fffd727c780406ed8f9136. 2024-12-03T20:34:24,107 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733258063204.8e57dc10d2fffd727c780406ed8f9136. after waiting 0 ms 2024-12-03T20:34:24,108 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733258063204.8e57dc10d2fffd727c780406ed8f9136. 2024-12-03T20:34:24,109 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733258063204.8e57dc10d2fffd727c780406ed8f9136. 2024-12-03T20:34:24,109 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8e57dc10d2fffd727c780406ed8f9136: Waiting for close lock at 1733258064107Disabling compacts and flushes for region at 1733258064107Disabling writes for close at 1733258064107Writing region close event to WAL at 1733258064109 (+2 ms)Closed at 1733258064109 2024-12-03T20:34:24,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741870_1047 (size=95) 2024-12-03T20:34:24,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741870_1047 (size=95) 2024-12-03T20:34:24,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741870_1047 (size=95) 2024-12-03T20:34:24,115 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T20:34:24,116 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733258063809) 2024-12-03T20:34:24,129 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=408 (was 406) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:54412 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:48562 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:55844 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=283 (was 273) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2126 (was 2333) 2024-12-03T20:34:24,141 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=408, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=283, ProcessCount=11, AvailableMemoryMB=2124 2024-12-03T20:34:24,155 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:24,161 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733258064155, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:24,162 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733258064162 2024-12-03T20:34:24,169 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733258064155/wal.1733258064162 2024-12-03T20:34:24,171 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:33459:33459)] 2024-12-03T20:34:24,172 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5afff658dd3b10ec7de4a067bf7fd021, NAME => 'testSequentialEditLogSeqNum,,1733258064156.5afff658dd3b10ec7de4a067bf7fd021.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:24,172 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733258064156.5afff658dd3b10ec7de4a067bf7fd021.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:24,173 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5afff658dd3b10ec7de4a067bf7fd021 2024-12-03T20:34:24,173 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5afff658dd3b10ec7de4a067bf7fd021 2024-12-03T20:34:24,173 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:39361/hbase/data/default/testSequentialEditLogSeqNum/5afff658dd3b10ec7de4a067bf7fd021 doesn't exist for region: 5afff658dd3b10ec7de4a067bf7fd021 on table testSequentialEditLogSeqNum 2024-12-03T20:34:24,174 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 5afff658dd3b10ec7de4a067bf7fd021 on table testSequentialEditLogSeqNum 2024-12-03T20:34:24,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741875_1052 (size=62) 2024-12-03T20:34:24,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741875_1052 (size=62) 2024-12-03T20:34:24,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741875_1052 (size=62) 2024-12-03T20:34:24,187 INFO [StoreOpener-5afff658dd3b10ec7de4a067bf7fd021-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5afff658dd3b10ec7de4a067bf7fd021 2024-12-03T20:34:24,189 INFO [StoreOpener-5afff658dd3b10ec7de4a067bf7fd021-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5afff658dd3b10ec7de4a067bf7fd021 columnFamilyName a 2024-12-03T20:34:24,190 DEBUG [StoreOpener-5afff658dd3b10ec7de4a067bf7fd021-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:24,190 INFO [StoreOpener-5afff658dd3b10ec7de4a067bf7fd021-1 {}] regionserver.HStore(327): Store=5afff658dd3b10ec7de4a067bf7fd021/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:24,191 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5afff658dd3b10ec7de4a067bf7fd021 2024-12-03T20:34:24,191 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testSequentialEditLogSeqNum/5afff658dd3b10ec7de4a067bf7fd021 2024-12-03T20:34:24,192 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testSequentialEditLogSeqNum/5afff658dd3b10ec7de4a067bf7fd021 2024-12-03T20:34:24,192 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5afff658dd3b10ec7de4a067bf7fd021 2024-12-03T20:34:24,192 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5afff658dd3b10ec7de4a067bf7fd021 2024-12-03T20:34:24,195 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5afff658dd3b10ec7de4a067bf7fd021 2024-12-03T20:34:24,197 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testSequentialEditLogSeqNum/5afff658dd3b10ec7de4a067bf7fd021/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T20:34:24,198 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5afff658dd3b10ec7de4a067bf7fd021; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67352744, jitterRate=0.0036340951919555664}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T20:34:24,199 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5afff658dd3b10ec7de4a067bf7fd021: Writing region info on filesystem at 1733258064173Initializing all the Stores at 1733258064186 (+13 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258064186Cleaning up temporary data from old regions at 1733258064192 (+6 ms)Region opened successfully at 1733258064198 (+6 ms) 2024-12-03T20:34:24,210 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5afff658dd3b10ec7de4a067bf7fd021 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-03T20:34:24,237 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testSequentialEditLogSeqNum/5afff658dd3b10ec7de4a067bf7fd021/.tmp/a/b5af1bdd67264d8ab68c464806f3d592 is 81, key is testSequentialEditLogSeqNum/a:x0/1733258064199/Put/seqid=0 2024-12-03T20:34:24,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741876_1053 (size=5833) 2024-12-03T20:34:24,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741876_1053 (size=5833) 2024-12-03T20:34:24,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741876_1053 (size=5833) 2024-12-03T20:34:24,245 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testSequentialEditLogSeqNum/5afff658dd3b10ec7de4a067bf7fd021/.tmp/a/b5af1bdd67264d8ab68c464806f3d592 2024-12-03T20:34:24,254 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testSequentialEditLogSeqNum/5afff658dd3b10ec7de4a067bf7fd021/.tmp/a/b5af1bdd67264d8ab68c464806f3d592 as hdfs://localhost:39361/hbase/data/default/testSequentialEditLogSeqNum/5afff658dd3b10ec7de4a067bf7fd021/a/b5af1bdd67264d8ab68c464806f3d592 2024-12-03T20:34:24,260 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testSequentialEditLogSeqNum/5afff658dd3b10ec7de4a067bf7fd021/a/b5af1bdd67264d8ab68c464806f3d592, entries=10, sequenceid=13, filesize=5.7 K 2024-12-03T20:34:24,262 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 5afff658dd3b10ec7de4a067bf7fd021 in 52ms, sequenceid=13, compaction requested=false 2024-12-03T20:34:24,262 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5afff658dd3b10ec7de4a067bf7fd021: 2024-12-03T20:34:24,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T20:34:24,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T20:34:24,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T20:34:24,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T20:34:24,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T20:34:24,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741874_1051 (size=1845) 2024-12-03T20:34:24,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741874_1051 (size=1845) 2024-12-03T20:34:24,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741874_1051 (size=1845) 2024-12-03T20:34:24,307 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39361/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733258064155/wal.1733258064162, size=1.8 K (1845bytes) 2024-12-03T20:34:24,307 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39361/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733258064155/wal.1733258064162 2024-12-03T20:34:24,308 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39361/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733258064155/wal.1733258064162 after 1ms 2024-12-03T20:34:24,311 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733258064155/wal.1733258064162: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:24,311 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39361/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733258064155/wal.1733258064162 took 4ms 2024-12-03T20:34:24,313 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39361/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733258064155/wal.1733258064162 so closing down 2024-12-03T20:34:24,314 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T20:34:24,314 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733258064162.temp 2024-12-03T20:34:24,316 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testSequentialEditLogSeqNum/5afff658dd3b10ec7de4a067bf7fd021/recovered.edits/0000000000000000003-wal.1733258064162.temp 2024-12-03T20:34:24,316 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T20:34:24,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741877_1054 (size=1477) 2024-12-03T20:34:24,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741877_1054 (size=1477) 2024-12-03T20:34:24,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741877_1054 (size=1477) 2024-12-03T20:34:24,329 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testSequentialEditLogSeqNum/5afff658dd3b10ec7de4a067bf7fd021/recovered.edits/0000000000000000003-wal.1733258064162.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-03T20:34:24,331 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39361/hbase/data/default/testSequentialEditLogSeqNum/5afff658dd3b10ec7de4a067bf7fd021/recovered.edits/0000000000000000003-wal.1733258064162.temp to hdfs://localhost:39361/hbase/data/default/testSequentialEditLogSeqNum/5afff658dd3b10ec7de4a067bf7fd021/recovered.edits/0000000000000000020 2024-12-03T20:34:24,331 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 20 ms; skipped=2; WAL=hdfs://localhost:39361/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733258064155/wal.1733258064162, size=1.8 K, length=1845, corrupted=false, cancelled=false 2024-12-03T20:34:24,331 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39361/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733258064155/wal.1733258064162, journal: Splitting hdfs://localhost:39361/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733258064155/wal.1733258064162, size=1.8 K (1845bytes) at 1733258064307Finishing writing output for hdfs://localhost:39361/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733258064155/wal.1733258064162 so closing down at 1733258064314 (+7 ms)Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testSequentialEditLogSeqNum/5afff658dd3b10ec7de4a067bf7fd021/recovered.edits/0000000000000000003-wal.1733258064162.temp at 1733258064316 (+2 ms)3 split writer threads finished at 1733258064316Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testSequentialEditLogSeqNum/5afff658dd3b10ec7de4a067bf7fd021/recovered.edits/0000000000000000003-wal.1733258064162.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733258064329 (+13 ms)Rename recovered edits hdfs://localhost:39361/hbase/data/default/testSequentialEditLogSeqNum/5afff658dd3b10ec7de4a067bf7fd021/recovered.edits/0000000000000000003-wal.1733258064162.temp to hdfs://localhost:39361/hbase/data/default/testSequentialEditLogSeqNum/5afff658dd3b10ec7de4a067bf7fd021/recovered.edits/0000000000000000020 at 1733258064331 (+2 ms)Processed 17 edits across 1 Regions in 20 ms; skipped=2; WAL=hdfs://localhost:39361/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733258064155/wal.1733258064162, size=1.8 K, length=1845, corrupted=false, cancelled=false at 1733258064331 2024-12-03T20:34:24,343 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=413 (was 408) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:54412 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:48562 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:55844 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=283 (was 283), ProcessCount=11 (was 11), AvailableMemoryMB=2050 (was 2124) 2024-12-03T20:34:24,354 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=413, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=283, ProcessCount=11, AvailableMemoryMB=2049 2024-12-03T20:34:24,367 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:24,369 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:24,401 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T20:34:24,404 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-59684471, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/hregion-59684471, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:24,417 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-59684471/hregion-59684471.1733258064404, exclude list is [], retry=0 2024-12-03T20:34:24,420 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:24,420 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:24,420 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:24,423 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-59684471/hregion-59684471.1733258064404 2024-12-03T20:34:24,423 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:33459:33459)] 2024-12-03T20:34:24,423 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5cdb757ff2822950b1b880a5b4a9fea3, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733258064367.5cdb757ff2822950b1b880a5b4a9fea3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39361/hbase 2024-12-03T20:34:24,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741879_1056 (size=70) 2024-12-03T20:34:24,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741879_1056 (size=70) 2024-12-03T20:34:24,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741879_1056 (size=70) 2024-12-03T20:34:24,435 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733258064367.5cdb757ff2822950b1b880a5b4a9fea3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:24,436 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:24,438 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cdb757ff2822950b1b880a5b4a9fea3 columnFamilyName a 2024-12-03T20:34:24,438 DEBUG [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:24,438 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] regionserver.HStore(327): Store=5cdb757ff2822950b1b880a5b4a9fea3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:24,439 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:24,440 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cdb757ff2822950b1b880a5b4a9fea3 columnFamilyName b 2024-12-03T20:34:24,440 DEBUG [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:24,440 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] regionserver.HStore(327): Store=5cdb757ff2822950b1b880a5b4a9fea3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:24,440 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:24,442 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cdb757ff2822950b1b880a5b4a9fea3 columnFamilyName c 2024-12-03T20:34:24,442 DEBUG [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:24,442 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] regionserver.HStore(327): Store=5cdb757ff2822950b1b880a5b4a9fea3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:24,443 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:24,443 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:24,444 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:24,445 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:24,445 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:24,445 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T20:34:24,447 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:24,449 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T20:34:24,450 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5cdb757ff2822950b1b880a5b4a9fea3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63688460, jitterRate=-0.0509679913520813}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T20:34:24,451 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5cdb757ff2822950b1b880a5b4a9fea3: Writing region info on filesystem at 1733258064435Initializing all the Stores at 1733258064436 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258064436Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258064436Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258064436Cleaning up temporary data from old regions at 1733258064445 (+9 ms)Region opened successfully at 1733258064451 (+6 ms) 2024-12-03T20:34:24,451 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5cdb757ff2822950b1b880a5b4a9fea3, disabling compactions & flushes 2024-12-03T20:34:24,451 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733258064367.5cdb757ff2822950b1b880a5b4a9fea3. 2024-12-03T20:34:24,451 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733258064367.5cdb757ff2822950b1b880a5b4a9fea3. 2024-12-03T20:34:24,451 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733258064367.5cdb757ff2822950b1b880a5b4a9fea3. after waiting 0 ms 2024-12-03T20:34:24,451 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733258064367.5cdb757ff2822950b1b880a5b4a9fea3. 2024-12-03T20:34:24,452 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733258064367.5cdb757ff2822950b1b880a5b4a9fea3. 2024-12-03T20:34:24,452 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5cdb757ff2822950b1b880a5b4a9fea3: Waiting for close lock at 1733258064451Disabling compacts and flushes for region at 1733258064451Disabling writes for close at 1733258064451Writing region close event to WAL at 1733258064452 (+1 ms)Closed at 1733258064452 2024-12-03T20:34:24,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741878_1055 (size=95) 2024-12-03T20:34:24,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741878_1055 (size=95) 2024-12-03T20:34:24,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741878_1055 (size=95) 2024-12-03T20:34:24,461 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T20:34:24,461 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-59684471:(num 1733258064404) 2024-12-03T20:34:24,462 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T20:34:24,464 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:24,478 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258064464, exclude list is [], retry=0 2024-12-03T20:34:24,481 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:24,481 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:24,481 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:24,483 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258064464 2024-12-03T20:34:24,484 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:34005:34005)] 2024-12-03T20:34:24,484 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5cdb757ff2822950b1b880a5b4a9fea3, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733258064367.5cdb757ff2822950b1b880a5b4a9fea3.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:24,485 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733258064367.5cdb757ff2822950b1b880a5b4a9fea3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:24,485 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:24,485 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:24,486 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:24,487 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cdb757ff2822950b1b880a5b4a9fea3 columnFamilyName a 2024-12-03T20:34:24,487 DEBUG [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:24,487 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] regionserver.HStore(327): Store=5cdb757ff2822950b1b880a5b4a9fea3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:24,488 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:24,488 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cdb757ff2822950b1b880a5b4a9fea3 columnFamilyName b 2024-12-03T20:34:24,488 DEBUG [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:24,489 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] regionserver.HStore(327): Store=5cdb757ff2822950b1b880a5b4a9fea3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:24,489 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:24,490 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cdb757ff2822950b1b880a5b4a9fea3 columnFamilyName c 2024-12-03T20:34:24,490 DEBUG [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:24,490 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] regionserver.HStore(327): Store=5cdb757ff2822950b1b880a5b4a9fea3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:24,490 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:24,491 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:24,492 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:24,493 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:24,493 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:24,493 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T20:34:24,495 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:24,495 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5cdb757ff2822950b1b880a5b4a9fea3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64644384, jitterRate=-0.03672361373901367}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T20:34:24,496 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5cdb757ff2822950b1b880a5b4a9fea3: Writing region info on filesystem at 1733258064485Initializing all the Stores at 1733258064486 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258064486Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258064486Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258064486Cleaning up temporary data from old regions at 1733258064493 (+7 ms)Region opened successfully at 1733258064496 (+3 ms) 2024-12-03T20:34:24,500 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733258064499/Put/seqid=0 2024-12-03T20:34:24,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741881_1058 (size=4826) 2024-12-03T20:34:24,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741881_1058 (size=4826) 2024-12-03T20:34:24,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741881_1058 (size=4826) 2024-12-03T20:34:24,506 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:39361/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 5cdb757ff2822950b1b880a5b4a9fea3/a 2024-12-03T20:34:24,513 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-03T20:34:24,514 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T20:34:24,514 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5cdb757ff2822950b1b880a5b4a9fea3: 2024-12-03T20:34:24,516 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/a/c8b78768e32843cebb732337876d2189_SeqId_3_ 2024-12-03T20:34:24,517 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:39361/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 5cdb757ff2822950b1b880a5b4a9fea3/a as hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/a/c8b78768e32843cebb732337876d2189_SeqId_3_ - updating store file list. 2024-12-03T20:34:24,522 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for c8b78768e32843cebb732337876d2189_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-03T20:34:24,523 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/a/c8b78768e32843cebb732337876d2189_SeqId_3_ into 5cdb757ff2822950b1b880a5b4a9fea3/a 2024-12-03T20:34:24,523 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:39361/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 5cdb757ff2822950b1b880a5b4a9fea3/a (new location: hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/a/c8b78768e32843cebb732337876d2189_SeqId_3_) 2024-12-03T20:34:24,566 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39361/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258064464, size=0 (0bytes) 2024-12-03T20:34:24,566 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:39361/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258064464 might be still open, length is 0 2024-12-03T20:34:24,566 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39361/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258064464 2024-12-03T20:34:24,566 WARN [IPC Server handler 4 on default port 39361 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258064464 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-03T20:34:24,567 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39361/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258064464 after 1ms 2024-12-03T20:34:27,564 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:48656 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33321:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48656 dst: /127.0.0.1:33321 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33321 remote=/127.0.0.1:48656]. Total timeout mills is 60000, 56967 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:27,565 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:56038 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:46119:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56038 dst: /127.0.0.1:46119 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:27,565 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:54516 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36623:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54516 dst: /127.0.0.1:36623 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:27,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741880_1059 (size=473) 2024-12-03T20:34:27,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741880_1059 (size=473) 2024-12-03T20:34:27,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741880_1059 (size=473) 2024-12-03T20:34:28,567 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39361/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258064464 after 4001ms 2024-12-03T20:34:28,571 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258064464: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:28,572 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39361/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258064464 took 4006ms 2024-12-03T20:34:28,574 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:39361/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258064464; continuing. 2024-12-03T20:34:28,574 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39361/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258064464 so closing down 2024-12-03T20:34:28,574 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T20:34:28,575 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733258064464.temp 2024-12-03T20:34:28,576 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/recovered.edits/0000000000000000005-wal.1733258064464.temp 2024-12-03T20:34:28,577 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T20:34:28,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741882_1060 (size=259) 2024-12-03T20:34:28,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741882_1060 (size=259) 2024-12-03T20:34:28,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741882_1060 (size=259) 2024-12-03T20:34:28,583 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/recovered.edits/0000000000000000005-wal.1733258064464.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T20:34:28,585 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/recovered.edits/0000000000000000005-wal.1733258064464.temp to hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/recovered.edits/0000000000000000005 2024-12-03T20:34:28,585 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 13 ms; skipped=1; WAL=hdfs://localhost:39361/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258064464, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T20:34:28,585 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39361/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258064464, journal: Splitting hdfs://localhost:39361/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258064464, size=0 (0bytes) at 1733258064566Finishing writing output for hdfs://localhost:39361/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258064464 so closing down at 1733258068574 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/recovered.edits/0000000000000000005-wal.1733258064464.temp at 1733258068576 (+2 ms)3 split writer threads finished at 1733258068577 (+1 ms)Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/recovered.edits/0000000000000000005-wal.1733258064464.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733258068583 (+6 ms)Rename recovered edits hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/recovered.edits/0000000000000000005-wal.1733258064464.temp to hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/recovered.edits/0000000000000000005 at 1733258068585 (+2 ms)Processed 2 edits across 1 Regions in 13 ms; skipped=1; WAL=hdfs://localhost:39361/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258064464, size=0, length=0, corrupted=false, cancelled=false at 1733258068585 2024-12-03T20:34:28,587 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39361/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258064464 to hdfs://localhost:39361/hbase/oldWALs/wal.1733258064464 2024-12-03T20:34:28,588 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/recovered.edits/0000000000000000005 2024-12-03T20:34:28,588 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T20:34:28,590 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:28,605 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258068591, exclude list is [], retry=0 2024-12-03T20:34:28,607 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:28,607 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:28,608 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:28,609 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258068591 2024-12-03T20:34:28,610 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:34005:34005)] 2024-12-03T20:34:28,610 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5cdb757ff2822950b1b880a5b4a9fea3, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733258064367.5cdb757ff2822950b1b880a5b4a9fea3.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:28,610 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733258064367.5cdb757ff2822950b1b880a5b4a9fea3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:28,610 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:28,610 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:28,612 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:28,612 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cdb757ff2822950b1b880a5b4a9fea3 columnFamilyName a 2024-12-03T20:34:28,612 DEBUG [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:28,619 DEBUG [StoreFileOpener-5cdb757ff2822950b1b880a5b4a9fea3-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for c8b78768e32843cebb732337876d2189_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-03T20:34:28,619 DEBUG [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/a/c8b78768e32843cebb732337876d2189_SeqId_3_ 2024-12-03T20:34:28,619 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] regionserver.HStore(327): Store=5cdb757ff2822950b1b880a5b4a9fea3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:28,619 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:28,620 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cdb757ff2822950b1b880a5b4a9fea3 columnFamilyName b 2024-12-03T20:34:28,620 DEBUG [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:28,621 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] regionserver.HStore(327): Store=5cdb757ff2822950b1b880a5b4a9fea3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:28,621 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:28,622 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cdb757ff2822950b1b880a5b4a9fea3 columnFamilyName c 2024-12-03T20:34:28,622 DEBUG [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:28,623 INFO [StoreOpener-5cdb757ff2822950b1b880a5b4a9fea3-1 {}] regionserver.HStore(327): Store=5cdb757ff2822950b1b880a5b4a9fea3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:28,623 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:28,624 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:28,625 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:28,626 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/recovered.edits/0000000000000000005 2024-12-03T20:34:28,629 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:28,630 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/recovered.edits/0000000000000000005 2024-12-03T20:34:28,630 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5cdb757ff2822950b1b880a5b4a9fea3 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-03T20:34:28,643 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/.tmp/a/a7ae1c99cc594ca59eac6d5228360b6f is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733258064528/Put/seqid=0 2024-12-03T20:34:28,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741884_1062 (size=5149) 2024-12-03T20:34:28,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741884_1062 (size=5149) 2024-12-03T20:34:28,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741884_1062 (size=5149) 2024-12-03T20:34:28,650 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/.tmp/a/a7ae1c99cc594ca59eac6d5228360b6f 2024-12-03T20:34:28,656 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/.tmp/a/a7ae1c99cc594ca59eac6d5228360b6f as hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/a/a7ae1c99cc594ca59eac6d5228360b6f 2024-12-03T20:34:28,661 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/a/a7ae1c99cc594ca59eac6d5228360b6f, entries=1, sequenceid=5, filesize=5.0 K 2024-12-03T20:34:28,661 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 5cdb757ff2822950b1b880a5b4a9fea3 in 31ms, sequenceid=5, compaction requested=false; wal=null 2024-12-03T20:34:28,662 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/recovered.edits/0000000000000000005 2024-12-03T20:34:28,663 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:28,663 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:28,664 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T20:34:28,666 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5cdb757ff2822950b1b880a5b4a9fea3 2024-12-03T20:34:28,668 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5cdb757ff2822950b1b880a5b4a9fea3/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-03T20:34:28,669 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5cdb757ff2822950b1b880a5b4a9fea3; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66532992, jitterRate=-0.008581161499023438}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T20:34:28,669 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5cdb757ff2822950b1b880a5b4a9fea3: Writing region info on filesystem at 1733258068610Initializing all the Stores at 1733258068611 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258068611Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258068611Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258068611Obtaining lock to block concurrent updates at 1733258068630 (+19 ms)Preparing flush snapshotting stores in 5cdb757ff2822950b1b880a5b4a9fea3 at 1733258068630Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733258064367.5cdb757ff2822950b1b880a5b4a9fea3., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733258068630Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733258064367.5cdb757ff2822950b1b880a5b4a9fea3. at 1733258068630Flushing 5cdb757ff2822950b1b880a5b4a9fea3/a: creating writer at 1733258068630Flushing 5cdb757ff2822950b1b880a5b4a9fea3/a: appending metadata at 1733258068643 (+13 ms)Flushing 5cdb757ff2822950b1b880a5b4a9fea3/a: closing flushed file at 1733258068643Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3edabe09: reopening flushed file at 1733258068655 (+12 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 5cdb757ff2822950b1b880a5b4a9fea3 in 31ms, sequenceid=5, compaction requested=false; wal=null at 1733258068661 (+6 ms)Cleaning up temporary data from old regions at 1733258068663 (+2 ms)Region opened successfully at 1733258068669 (+6 ms) 2024-12-03T20:34:28,672 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5cdb757ff2822950b1b880a5b4a9fea3, disabling compactions & flushes 2024-12-03T20:34:28,672 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733258064367.5cdb757ff2822950b1b880a5b4a9fea3. 2024-12-03T20:34:28,672 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733258064367.5cdb757ff2822950b1b880a5b4a9fea3. 2024-12-03T20:34:28,672 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733258064367.5cdb757ff2822950b1b880a5b4a9fea3. after waiting 0 ms 2024-12-03T20:34:28,673 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733258064367.5cdb757ff2822950b1b880a5b4a9fea3. 2024-12-03T20:34:28,673 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733258064367.5cdb757ff2822950b1b880a5b4a9fea3. 2024-12-03T20:34:28,673 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5cdb757ff2822950b1b880a5b4a9fea3: Waiting for close lock at 1733258068672Disabling compacts and flushes for region at 1733258068672Disabling writes for close at 1733258068672Writing region close event to WAL at 1733258068673 (+1 ms)Closed at 1733258068673 2024-12-03T20:34:28,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741883_1061 (size=95) 2024-12-03T20:34:28,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741883_1061 (size=95) 2024-12-03T20:34:28,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741883_1061 (size=95) 2024-12-03T20:34:28,678 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T20:34:28,678 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733258068591) 2024-12-03T20:34:28,689 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=419 (was 413) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39159 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_113683374_22 at /127.0.0.1:54550 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:39361 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:39361 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_113683374_22 at /127.0.0.1:48670 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_113683374_22 at /127.0.0.1:56074 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:39159 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=261 (was 283), ProcessCount=11 (was 11), AvailableMemoryMB=2058 (was 2049) - AvailableMemoryMB LEAK? - 2024-12-03T20:34:28,700 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=419, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=261, ProcessCount=11, AvailableMemoryMB=2057 2024-12-03T20:34:28,715 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:28,720 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T20:34:28,723 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is b29c245002d9,38205,1733258053478 2024-12-03T20:34:28,725 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@59117651 2024-12-03T20:34:28,726 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T20:34:28,728 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T20:34:28,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38205 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T20:34:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38205 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-03T20:34:28,739 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T20:34:28,741 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38205 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-03T20:34:28,741 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:28,743 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T20:34:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T20:34:28,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741885_1063 (size=694) 2024-12-03T20:34:28,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741885_1063 (size=694) 2024-12-03T20:34:28,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741885_1063 (size=694) 2024-12-03T20:34:28,756 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2fcbcf91e026c1e68ac208aa693d488a, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596 2024-12-03T20:34:28,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741886_1064 (size=77) 2024-12-03T20:34:28,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741886_1064 (size=77) 2024-12-03T20:34:28,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741886_1064 (size=77) 2024-12-03T20:34:28,765 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:28,765 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 2fcbcf91e026c1e68ac208aa693d488a, disabling compactions & flushes 2024-12-03T20:34:28,765 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:28,765 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:28,765 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. after waiting 0 ms 2024-12-03T20:34:28,765 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:28,765 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:28,765 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2fcbcf91e026c1e68ac208aa693d488a: Waiting for close lock at 1733258068765Disabling compacts and flushes for region at 1733258068765Disabling writes for close at 1733258068765Writing region close event to WAL at 1733258068765Closed at 1733258068765 2024-12-03T20:34:28,767 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T20:34:28,774 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733258068768"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733258068768"}]},"ts":"1733258068768"} 2024-12-03T20:34:28,777 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T20:34:28,778 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T20:34:28,780 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733258068778"}]},"ts":"1733258068778"} 2024-12-03T20:34:28,784 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-03T20:34:28,784 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(204): Hosts are {b29c245002d9=0} racks are {/default-rack=0} 2024-12-03T20:34:28,785 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T20:34:28,785 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T20:34:28,785 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-03T20:34:28,785 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T20:34:28,785 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T20:34:28,785 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-03T20:34:28,785 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T20:34:28,785 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T20:34:28,785 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-03T20:34:28,786 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T20:34:28,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, ASSIGN}] 2024-12-03T20:34:28,788 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, ASSIGN 2024-12-03T20:34:28,790 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, ASSIGN; state=OFFLINE, location=b29c245002d9,35007,1733258054368; forceNewPlan=false, retain=false 2024-12-03T20:34:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T20:34:28,946 INFO [b29c245002d9:38205 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T20:34:28,947 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2fcbcf91e026c1e68ac208aa693d488a, regionState=OPENING, regionLocation=b29c245002d9,35007,1733258054368 2024-12-03T20:34:28,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, ASSIGN because future has completed 2024-12-03T20:34:28,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2fcbcf91e026c1e68ac208aa693d488a, server=b29c245002d9,35007,1733258054368}] 2024-12-03T20:34:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T20:34:29,105 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T20:34:29,108 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51699, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T20:34:29,115 INFO [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:29,116 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 2fcbcf91e026c1e68ac208aa693d488a, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:29,116 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,116 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:29,116 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,117 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,118 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,120 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fcbcf91e026c1e68ac208aa693d488a columnFamilyName cf1 2024-12-03T20:34:29,120 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:29,121 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(327): Store=2fcbcf91e026c1e68ac208aa693d488a/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:29,121 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,122 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fcbcf91e026c1e68ac208aa693d488a columnFamilyName cf2 2024-12-03T20:34:29,123 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:29,123 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(327): Store=2fcbcf91e026c1e68ac208aa693d488a/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:29,123 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,124 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,125 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,126 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,126 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,127 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T20:34:29,129 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,131 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T20:34:29,132 INFO [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 2fcbcf91e026c1e68ac208aa693d488a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59212447, jitterRate=-0.11766578257083893}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T20:34:29,132 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,133 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 2fcbcf91e026c1e68ac208aa693d488a: Running coprocessor pre-open hook at 1733258069117Writing region info on filesystem at 1733258069117Initializing all the Stores at 1733258069118 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258069118Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258069118Cleaning up temporary data from old regions at 1733258069126 (+8 ms)Running coprocessor post-open hooks at 1733258069132 (+6 ms)Region opened successfully at 1733258069133 (+1 ms) 2024-12-03T20:34:29,135 INFO [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., pid=6, masterSystemTime=1733258069104 2024-12-03T20:34:29,137 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:29,137 INFO [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:29,138 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2fcbcf91e026c1e68ac208aa693d488a, regionState=OPEN, openSeqNum=2, regionLocation=b29c245002d9,35007,1733258054368 2024-12-03T20:34:29,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2fcbcf91e026c1e68ac208aa693d488a, server=b29c245002d9,35007,1733258054368 because future has completed 2024-12-03T20:34:29,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T20:34:29,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 2fcbcf91e026c1e68ac208aa693d488a, server=b29c245002d9,35007,1733258054368 in 192 msec 2024-12-03T20:34:29,149 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T20:34:29,149 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, ASSIGN in 358 msec 2024-12-03T20:34:29,150 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T20:34:29,151 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733258069150"}]},"ts":"1733258069150"} 2024-12-03T20:34:29,153 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-03T20:34:29,155 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T20:34:29,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 423 msec 2024-12-03T20:34:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38205 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T20:34:29,375 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-03T20:34:29,375 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-03T20:34:29,377 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T20:34:29,383 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-03T20:34:29,383 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T20:34:29,384 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-03T20:34:29,394 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35007,1733258054368, seqNum=2] 2024-12-03T20:34:29,395 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T20:34:29,396 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T20:34:29,410 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38205 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=2fcbcf91e026c1e68ac208aa693d488a, source=b29c245002d9,35007,1733258054368, destination=b29c245002d9,35249,1733258054237, warming up region on b29c245002d9,35249,1733258054237 2024-12-03T20:34:29,412 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38205 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=2fcbcf91e026c1e68ac208aa693d488a, source=b29c245002d9,35007,1733258054368, destination=b29c245002d9,35249,1733258054237, running balancer 2024-12-03T20:34:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38205 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, REOPEN/MOVE 2024-12-03T20:34:29,413 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, REOPEN/MOVE 2024-12-03T20:34:29,415 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2fcbcf91e026c1e68ac208aa693d488a, regionState=CLOSING, regionLocation=b29c245002d9,35007,1733258054368 2024-12-03T20:34:29,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35249 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35249 {}] regionserver.HRegion(7855): Warmup {ENCODED => 2fcbcf91e026c1e68ac208aa693d488a, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35249 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:29,417 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,418 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fcbcf91e026c1e68ac208aa693d488a columnFamilyName cf1 2024-12-03T20:34:29,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, REOPEN/MOVE because future has completed 2024-12-03T20:34:29,418 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:29,419 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T20:34:29,419 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2fcbcf91e026c1e68ac208aa693d488a, server=b29c245002d9,35007,1733258054368}] 2024-12-03T20:34:29,420 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(327): Store=2fcbcf91e026c1e68ac208aa693d488a/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:29,420 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,421 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fcbcf91e026c1e68ac208aa693d488a columnFamilyName cf2 2024-12-03T20:34:29,421 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:29,422 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(327): Store=2fcbcf91e026c1e68ac208aa693d488a/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35249 {}] regionserver.HRegion(1722): Closing 2fcbcf91e026c1e68ac208aa693d488a, disabling compactions & flushes 2024-12-03T20:34:29,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35249 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35249 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35249 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. after waiting 0 ms 2024-12-03T20:34:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35249 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:29,424 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35249 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35249 {}] regionserver.HRegion(1676): Region close journal for 2fcbcf91e026c1e68ac208aa693d488a: Waiting for close lock at 1733258069422Disabling compacts and flushes for region at 1733258069422Disabling writes for close at 1733258069422Writing region close event to WAL at 1733258069424 (+2 ms)Closed at 1733258069424 2024-12-03T20:34:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38205 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-03T20:34:29,579 INFO [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,579 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T20:34:29,580 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 2fcbcf91e026c1e68ac208aa693d488a, disabling compactions & flushes 2024-12-03T20:34:29,580 INFO [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:29,580 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:29,580 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. after waiting 0 ms 2024-12-03T20:34:29,580 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:29,580 INFO [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 2fcbcf91e026c1e68ac208aa693d488a 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-03T20:34:29,601 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/.tmp/cf1/ee904b7e01eb4648ba309d0bd0128cd0 is 35, key is r1/cf1:q/1733258069397/Put/seqid=0 2024-12-03T20:34:29,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741887_1065 (size=4783) 2024-12-03T20:34:29,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741887_1065 (size=4783) 2024-12-03T20:34:29,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741887_1065 (size=4783) 2024-12-03T20:34:29,609 INFO [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/.tmp/cf1/ee904b7e01eb4648ba309d0bd0128cd0 2024-12-03T20:34:29,619 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/.tmp/cf1/ee904b7e01eb4648ba309d0bd0128cd0 as hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf1/ee904b7e01eb4648ba309d0bd0128cd0 2024-12-03T20:34:29,627 INFO [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf1/ee904b7e01eb4648ba309d0bd0128cd0, entries=1, sequenceid=5, filesize=4.7 K 2024-12-03T20:34:29,629 INFO [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 2fcbcf91e026c1e68ac208aa693d488a in 48ms, sequenceid=5, compaction requested=false 2024-12-03T20:34:29,629 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-03T20:34:29,635 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-03T20:34:29,638 INFO [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:29,638 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 2fcbcf91e026c1e68ac208aa693d488a: Waiting for close lock at 1733258069580Running coprocessor pre-close hooks at 1733258069580Disabling compacts and flushes for region at 1733258069580Disabling writes for close at 1733258069580Obtaining lock to block concurrent updates at 1733258069580Preparing flush snapshotting stores in 2fcbcf91e026c1e68ac208aa693d488a at 1733258069580Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733258069580Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. at 1733258069581 (+1 ms)Flushing 2fcbcf91e026c1e68ac208aa693d488a/cf1: creating writer at 1733258069581Flushing 2fcbcf91e026c1e68ac208aa693d488a/cf1: appending metadata at 1733258069601 (+20 ms)Flushing 2fcbcf91e026c1e68ac208aa693d488a/cf1: closing flushed file at 1733258069601Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10646625: reopening flushed file at 1733258069617 (+16 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 2fcbcf91e026c1e68ac208aa693d488a in 48ms, sequenceid=5, compaction requested=false at 1733258069629 (+12 ms)Writing region close event to WAL at 1733258069630 (+1 ms)Running coprocessor post-close hooks at 1733258069636 (+6 ms)Closed at 1733258069638 (+2 ms) 2024-12-03T20:34:29,638 INFO [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 2fcbcf91e026c1e68ac208aa693d488a move to b29c245002d9,35249,1733258054237 record at close sequenceid=5 2024-12-03T20:34:29,642 INFO [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,643 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2fcbcf91e026c1e68ac208aa693d488a, regionState=CLOSED 2024-12-03T20:34:29,646 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2fcbcf91e026c1e68ac208aa693d488a, server=b29c245002d9,35007,1733258054368 because future has completed 2024-12-03T20:34:29,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T20:34:29,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 2fcbcf91e026c1e68ac208aa693d488a, server=b29c245002d9,35007,1733258054368 in 229 msec 2024-12-03T20:34:29,653 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, REOPEN/MOVE; state=CLOSED, location=b29c245002d9,35249,1733258054237; forceNewPlan=false, retain=false 2024-12-03T20:34:29,803 INFO [b29c245002d9:38205 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T20:34:29,804 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2fcbcf91e026c1e68ac208aa693d488a, regionState=OPENING, regionLocation=b29c245002d9,35249,1733258054237 2024-12-03T20:34:29,812 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, REOPEN/MOVE because future has completed 2024-12-03T20:34:29,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2fcbcf91e026c1e68ac208aa693d488a, server=b29c245002d9,35249,1733258054237}] 2024-12-03T20:34:29,972 INFO [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:29,973 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 2fcbcf91e026c1e68ac208aa693d488a, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:29,973 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,973 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:29,974 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,974 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,976 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,978 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fcbcf91e026c1e68ac208aa693d488a columnFamilyName cf1 2024-12-03T20:34:29,978 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:29,986 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf1/ee904b7e01eb4648ba309d0bd0128cd0 2024-12-03T20:34:29,987 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(327): Store=2fcbcf91e026c1e68ac208aa693d488a/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:29,987 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,988 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fcbcf91e026c1e68ac208aa693d488a columnFamilyName cf2 2024-12-03T20:34:29,988 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:29,989 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(327): Store=2fcbcf91e026c1e68ac208aa693d488a/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:29,989 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,990 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,991 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,992 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,992 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,993 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T20:34:29,995 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,996 INFO [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 2fcbcf91e026c1e68ac208aa693d488a; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73412205, jitterRate=0.09392710030078888}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T20:34:29,996 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:29,997 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 2fcbcf91e026c1e68ac208aa693d488a: Running coprocessor pre-open hook at 1733258069974Writing region info on filesystem at 1733258069974Initializing all the Stores at 1733258069975 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258069976 (+1 ms)Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258069976Cleaning up temporary data from old regions at 1733258069992 (+16 ms)Running coprocessor post-open hooks at 1733258069996 (+4 ms)Region opened successfully at 1733258069996 2024-12-03T20:34:29,998 INFO [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., pid=9, masterSystemTime=1733258069967 2024-12-03T20:34:30,000 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:30,001 INFO [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:30,001 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=2fcbcf91e026c1e68ac208aa693d488a, regionState=OPEN, openSeqNum=9, regionLocation=b29c245002d9,35249,1733258054237 2024-12-03T20:34:30,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2fcbcf91e026c1e68ac208aa693d488a, server=b29c245002d9,35249,1733258054237 because future has completed 2024-12-03T20:34:30,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-03T20:34:30,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 2fcbcf91e026c1e68ac208aa693d488a, server=b29c245002d9,35249,1733258054237 in 192 msec 2024-12-03T20:34:30,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, REOPEN/MOVE in 596 msec 2024-12-03T20:34:30,019 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T20:34:30,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T20:34:30,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35007 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:54468 deadline: 1733258130024, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b29c245002d9 port=35249 startCode=1733258054237. As of locationSeqNum=5. 2024-12-03T20:34:30,042 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35007,1733258054368, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35007,1733258054368, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b29c245002d9 port=35249 startCode=1733258054237. As of locationSeqNum=5. 2024-12-03T20:34:30,043 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35007,1733258054368, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b29c245002d9 port=35249 startCode=1733258054237. As of locationSeqNum=5. 2024-12-03T20:34:30,043 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35007,1733258054368, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35249,1733258054237, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b29c245002d9 port=35249 startCode=1733258054237. As of locationSeqNum=5. 2024-12-03T20:34:30,165 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2fcbcf91e026c1e68ac208aa693d488a 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-03T20:34:30,180 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/.tmp/cf1/01bde4e7c8524bc6bb32d584448d186d is 29, key is r1/cf1:/1733258070155/DeleteFamily/seqid=0 2024-12-03T20:34:30,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741888_1066 (size=4906) 2024-12-03T20:34:30,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741888_1066 (size=4906) 2024-12-03T20:34:30,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741888_1066 (size=4906) 2024-12-03T20:34:30,187 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/.tmp/cf1/01bde4e7c8524bc6bb32d584448d186d 2024-12-03T20:34:30,193 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 01bde4e7c8524bc6bb32d584448d186d 2024-12-03T20:34:30,212 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/.tmp/cf2/328671b543e24e32a3fd6f3ac3ce4366 is 29, key is r1/cf2:/1733258070155/DeleteFamily/seqid=0 2024-12-03T20:34:30,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741889_1067 (size=4906) 2024-12-03T20:34:30,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741889_1067 (size=4906) 2024-12-03T20:34:30,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741889_1067 (size=4906) 2024-12-03T20:34:30,220 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/.tmp/cf2/328671b543e24e32a3fd6f3ac3ce4366 2024-12-03T20:34:30,226 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 328671b543e24e32a3fd6f3ac3ce4366 2024-12-03T20:34:30,227 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/.tmp/cf1/01bde4e7c8524bc6bb32d584448d186d as hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf1/01bde4e7c8524bc6bb32d584448d186d 2024-12-03T20:34:30,234 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 01bde4e7c8524bc6bb32d584448d186d 2024-12-03T20:34:30,234 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf1/01bde4e7c8524bc6bb32d584448d186d, entries=1, sequenceid=12, filesize=4.8 K 2024-12-03T20:34:30,236 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/.tmp/cf2/328671b543e24e32a3fd6f3ac3ce4366 as hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf2/328671b543e24e32a3fd6f3ac3ce4366 2024-12-03T20:34:30,242 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 328671b543e24e32a3fd6f3ac3ce4366 2024-12-03T20:34:30,242 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf2/328671b543e24e32a3fd6f3ac3ce4366, entries=1, sequenceid=12, filesize=4.8 K 2024-12-03T20:34:30,243 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 2fcbcf91e026c1e68ac208aa693d488a in 78ms, sequenceid=12, compaction requested=false 2024-12-03T20:34:30,243 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2fcbcf91e026c1e68ac208aa693d488a: 2024-12-03T20:34:30,246 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T20:34:30,247 DEBUG [Time-limited test {}] regionserver.HStore(1541): 2fcbcf91e026c1e68ac208aa693d488a/cf1 is initiating major compaction (all files) 2024-12-03T20:34:30,248 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T20:34:30,248 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:30,248 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 2fcbcf91e026c1e68ac208aa693d488a/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:30,249 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf1/ee904b7e01eb4648ba309d0bd0128cd0, hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf1/01bde4e7c8524bc6bb32d584448d186d] into tmpdir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/.tmp, totalSize=9.5 K 2024-12-03T20:34:30,250 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ee904b7e01eb4648ba309d0bd0128cd0, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733258069397 2024-12-03T20:34:30,251 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 01bde4e7c8524bc6bb32d584448d186d, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-03T20:34:30,264 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 2fcbcf91e026c1e68ac208aa693d488a#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T20:34:30,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741890_1068 (size=4626) 2024-12-03T20:34:30,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741890_1068 (size=4626) 2024-12-03T20:34:30,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741890_1068 (size=4626) 2024-12-03T20:34:30,282 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/.tmp/cf1/981cc8d8e7e84a4c9cb43e2bd76e3045 as hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf1/981cc8d8e7e84a4c9cb43e2bd76e3045 2024-12-03T20:34:30,299 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 2fcbcf91e026c1e68ac208aa693d488a/cf1 of 2fcbcf91e026c1e68ac208aa693d488a into 981cc8d8e7e84a4c9cb43e2bd76e3045(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T20:34:30,299 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 2fcbcf91e026c1e68ac208aa693d488a: 2024-12-03T20:34:30,300 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-03T20:34:30,300 DEBUG [Time-limited test {}] regionserver.HStore(1541): 2fcbcf91e026c1e68ac208aa693d488a/cf2 is initiating major compaction (all files) 2024-12-03T20:34:30,300 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T20:34:30,300 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T20:34:30,300 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 2fcbcf91e026c1e68ac208aa693d488a/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:30,300 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf2/328671b543e24e32a3fd6f3ac3ce4366] into tmpdir=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/.tmp, totalSize=4.8 K 2024-12-03T20:34:30,301 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 328671b543e24e32a3fd6f3ac3ce4366, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-03T20:34:30,309 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 2fcbcf91e026c1e68ac208aa693d488a#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T20:34:30,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741891_1069 (size=4592) 2024-12-03T20:34:30,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741891_1069 (size=4592) 2024-12-03T20:34:30,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741891_1069 (size=4592) 2024-12-03T20:34:30,328 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/.tmp/cf2/25a1125973414265b47641a4322e8e39 as hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf2/25a1125973414265b47641a4322e8e39 2024-12-03T20:34:30,336 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 2fcbcf91e026c1e68ac208aa693d488a/cf2 of 2fcbcf91e026c1e68ac208aa693d488a into 25a1125973414265b47641a4322e8e39(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T20:34:30,336 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 2fcbcf91e026c1e68ac208aa693d488a: 2024-12-03T20:34:30,342 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38205 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=2fcbcf91e026c1e68ac208aa693d488a, source=b29c245002d9,35249,1733258054237, destination=b29c245002d9,35007,1733258054368, warming up region on b29c245002d9,35007,1733258054368 2024-12-03T20:34:30,342 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38205 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=2fcbcf91e026c1e68ac208aa693d488a, source=b29c245002d9,35249,1733258054237, destination=b29c245002d9,35007,1733258054368, running balancer 2024-12-03T20:34:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38205 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, REOPEN/MOVE 2024-12-03T20:34:30,344 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, REOPEN/MOVE 2024-12-03T20:34:30,346 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=2fcbcf91e026c1e68ac208aa693d488a, regionState=CLOSING, regionLocation=b29c245002d9,35249,1733258054237 2024-12-03T20:34:30,346 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35007 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35007 {}] regionserver.HRegion(7855): Warmup {ENCODED => 2fcbcf91e026c1e68ac208aa693d488a, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35007 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:30,347 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:30,348 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fcbcf91e026c1e68ac208aa693d488a columnFamilyName cf1 2024-12-03T20:34:30,348 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:30,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, REOPEN/MOVE because future has completed 2024-12-03T20:34:30,349 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T20:34:30,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2fcbcf91e026c1e68ac208aa693d488a, server=b29c245002d9,35249,1733258054237}] 2024-12-03T20:34:30,357 INFO [StoreFileOpener-2fcbcf91e026c1e68ac208aa693d488a-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 01bde4e7c8524bc6bb32d584448d186d 2024-12-03T20:34:30,357 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf1/01bde4e7c8524bc6bb32d584448d186d 2024-12-03T20:34:30,363 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf1/981cc8d8e7e84a4c9cb43e2bd76e3045 2024-12-03T20:34:30,369 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf1/ee904b7e01eb4648ba309d0bd0128cd0 2024-12-03T20:34:30,369 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(327): Store=2fcbcf91e026c1e68ac208aa693d488a/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:30,370 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:30,371 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fcbcf91e026c1e68ac208aa693d488a columnFamilyName cf2 2024-12-03T20:34:30,371 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:30,378 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf2/25a1125973414265b47641a4322e8e39 2024-12-03T20:34:30,384 INFO [StoreFileOpener-2fcbcf91e026c1e68ac208aa693d488a-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 328671b543e24e32a3fd6f3ac3ce4366 2024-12-03T20:34:30,384 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf2/328671b543e24e32a3fd6f3ac3ce4366 2024-12-03T20:34:30,384 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(327): Store=2fcbcf91e026c1e68ac208aa693d488a/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:30,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35007 {}] regionserver.HRegion(1722): Closing 2fcbcf91e026c1e68ac208aa693d488a, disabling compactions & flushes 2024-12-03T20:34:30,385 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35007 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:30,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35007 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:30,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35007 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. after waiting 0 ms 2024-12-03T20:34:30,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35007 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:30,386 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35007 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:30,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35007 {}] regionserver.HRegion(1676): Region close journal for 2fcbcf91e026c1e68ac208aa693d488a: Waiting for close lock at 1733258070384Disabling compacts and flushes for region at 1733258070384Disabling writes for close at 1733258070385 (+1 ms)Writing region close event to WAL at 1733258070386 (+1 ms)Closed at 1733258070386 2024-12-03T20:34:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38205 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-03T20:34:30,504 INFO [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:30,504 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T20:34:30,504 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 2fcbcf91e026c1e68ac208aa693d488a, disabling compactions & flushes 2024-12-03T20:34:30,504 INFO [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:30,504 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:30,504 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. after waiting 0 ms 2024-12-03T20:34:30,504 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:30,505 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf1/ee904b7e01eb4648ba309d0bd0128cd0, hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf1/01bde4e7c8524bc6bb32d584448d186d] to archive 2024-12-03T20:34:30,508 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T20:34:30,513 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf1/ee904b7e01eb4648ba309d0bd0128cd0 to hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf1/ee904b7e01eb4648ba309d0bd0128cd0 2024-12-03T20:34:30,515 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf1/01bde4e7c8524bc6bb32d584448d186d to hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf1/01bde4e7c8524bc6bb32d584448d186d 2024-12-03T20:34:30,528 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf2/328671b543e24e32a3fd6f3ac3ce4366] to archive 2024-12-03T20:34:30,529 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T20:34:30,531 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf2/328671b543e24e32a3fd6f3ac3ce4366 to hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf2/328671b543e24e32a3fd6f3ac3ce4366 2024-12-03T20:34:30,536 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-03T20:34:30,537 INFO [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:30,537 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 2fcbcf91e026c1e68ac208aa693d488a: Waiting for close lock at 1733258070504Running coprocessor pre-close hooks at 1733258070504Disabling compacts and flushes for region at 1733258070504Disabling writes for close at 1733258070504Writing region close event to WAL at 1733258070532 (+28 ms)Running coprocessor post-close hooks at 1733258070537 (+5 ms)Closed at 1733258070537 2024-12-03T20:34:30,537 INFO [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 2fcbcf91e026c1e68ac208aa693d488a move to b29c245002d9,35007,1733258054368 record at close sequenceid=12 2024-12-03T20:34:30,540 INFO [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:30,541 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=2fcbcf91e026c1e68ac208aa693d488a, regionState=CLOSED 2024-12-03T20:34:30,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2fcbcf91e026c1e68ac208aa693d488a, server=b29c245002d9,35249,1733258054237 because future has completed 2024-12-03T20:34:30,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-03T20:34:30,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 2fcbcf91e026c1e68ac208aa693d488a, server=b29c245002d9,35249,1733258054237 in 198 msec 2024-12-03T20:34:30,552 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, REOPEN/MOVE; state=CLOSED, location=b29c245002d9,35007,1733258054368; forceNewPlan=false, retain=false 2024-12-03T20:34:30,702 INFO [b29c245002d9:38205 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T20:34:30,703 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=2fcbcf91e026c1e68ac208aa693d488a, regionState=OPENING, regionLocation=b29c245002d9,35007,1733258054368 2024-12-03T20:34:30,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, REOPEN/MOVE because future has completed 2024-12-03T20:34:30,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2fcbcf91e026c1e68ac208aa693d488a, server=b29c245002d9,35007,1733258054368}] 2024-12-03T20:34:30,866 INFO [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:30,867 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 2fcbcf91e026c1e68ac208aa693d488a, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:30,868 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:30,868 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:30,868 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:30,868 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:30,872 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:30,873 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fcbcf91e026c1e68ac208aa693d488a columnFamilyName cf1 2024-12-03T20:34:30,874 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:30,882 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf1/981cc8d8e7e84a4c9cb43e2bd76e3045 2024-12-03T20:34:30,882 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(327): Store=2fcbcf91e026c1e68ac208aa693d488a/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:30,882 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:30,883 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fcbcf91e026c1e68ac208aa693d488a columnFamilyName cf2 2024-12-03T20:34:30,883 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:30,890 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf2/25a1125973414265b47641a4322e8e39 2024-12-03T20:34:30,890 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(327): Store=2fcbcf91e026c1e68ac208aa693d488a/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:30,891 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:30,891 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:30,893 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:30,894 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:30,894 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:30,894 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T20:34:30,896 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:30,897 INFO [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 2fcbcf91e026c1e68ac208aa693d488a; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62473026, jitterRate=-0.06907936930656433}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T20:34:30,897 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:30,898 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 2fcbcf91e026c1e68ac208aa693d488a: Running coprocessor pre-open hook at 1733258070869Writing region info on filesystem at 1733258070869Initializing all the Stores at 1733258070871 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258070871Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258070871Cleaning up temporary data from old regions at 1733258070894 (+23 ms)Running coprocessor post-open hooks at 1733258070897 (+3 ms)Region opened successfully at 1733258070898 (+1 ms) 2024-12-03T20:34:30,899 INFO [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., pid=12, masterSystemTime=1733258070859 2024-12-03T20:34:30,901 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:30,901 INFO [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:30,902 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=2fcbcf91e026c1e68ac208aa693d488a, regionState=OPEN, openSeqNum=18, regionLocation=b29c245002d9,35007,1733258054368 2024-12-03T20:34:30,903 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38205 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=b29c245002d9,35007,1733258054368, table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-03T20:34:30,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2fcbcf91e026c1e68ac208aa693d488a, server=b29c245002d9,35007,1733258054368 because future has completed 2024-12-03T20:34:30,908 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-03T20:34:30,908 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 2fcbcf91e026c1e68ac208aa693d488a, server=b29c245002d9,35007,1733258054368 in 200 msec 2024-12-03T20:34:30,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, REOPEN/MOVE in 566 msec 2024-12-03T20:34:30,945 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T20:34:30,947 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T20:34:30,949 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server b29c245002d9,35007,1733258054368: testing ***** 2024-12-03T20:34:30,949 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-03T20:34:30,951 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-03T20:34:30,952 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-03T20:34:30,955 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-03T20:34:30,956 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-03T20:34:30,966 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 329359360 }, "NonHeapMemoryUsage": { "committed": 170917888, "init": 7667712, "max": -1, "used": 168386024 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "b29c245002d9", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 8, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 4, "ProcessCallTime_75th_percentile": 6, "ProcessCallTime_90th_percentile": 7, "ProcessCallTime_95th_percentile": 7, "ProcessCallTime_98th_percentile": 7, "ProcessCallTime_99th_percentile": 7, "ProcessCallTime_99.9th_percentile": 7, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 9, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 8, "TotalCallTime_95th_percentile": 8, "TotalCallTime_98th_percentile": 8, "TotalCallTime_99th_percentile": 8, "TotalCallTime_99.9th_percentile": 8, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "b29c245002d9", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:49830", "tag.serverName": "b29c245002d9,35249,1733258054237", "tag.clusterId": "3811d7df-5f5c-4ed7-96e1-56c5b559222b", "tag.Context": "regionserver", "tag.Hostname": "b29c245002d9", "regionCount": 1, "storeCount": 4, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733258054237, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 0.4, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 199680, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 1, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 4, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 4, "ScanTime_min": 0, "ScanTime_max": 2, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 1, "ScanTime_95th_percentile": 1, "ScanTime_98th_percentile": 1, "ScanTime_99th_percentile": 1, "ScanTime_99.9th_percentile": 1, "ScanTime_TimeRangeCount_0-1": 4, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 2, "Put_min": 1, "Put_max": 7, "Put_mean": 4, "Put_25th_percentile": 2, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 2, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 1, "Get_min": 2, "Get_max": 2, "Get_mean": 2, "Get_25th_percentile": 2, "Get_median": 2, "Get_75th_percentile": 2, "Get_90th_percentile": 2, "Get_95th_percentile": 2, "Get_98th_percentile": 2, "Get_99th_percentile": 2, "Get_99.9th_percentile": 2, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 4, "ScanSize_min": 0, "ScanSize_max": 144, "ScanSize_mean": 72, "ScanSize_25th_percentile": 36, "ScanSize_median": 72, "ScanSize_75th_percentile": 108, "ScanSize_90th_percentile": 129, "ScanSize_95th_percentile": 136, "ScanSize_98th_percentile": 141, "ScanSize_99th_percentile": 142, "ScanSize_99.9th_percentile": 143, "ScanSize_SizeRangeCount_0-10": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-03T20:34:30,969 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38205 {}] master.MasterRpcServices(700): b29c245002d9,35007,1733258054368 reported a fatal error: ***** ABORTING region server b29c245002d9,35007,1733258054368: testing ***** 2024-12-03T20:34:30,971 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b29c245002d9,35007,1733258054368' ***** 2024-12-03T20:34:30,971 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-03T20:34:30,971 INFO [RS:2;b29c245002d9:35007 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T20:34:30,971 INFO [RS:2;b29c245002d9:35007 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-03T20:34:30,971 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T20:34:30,972 INFO [RS:2;b29c245002d9:35007 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-03T20:34:30,972 INFO [RS:2;b29c245002d9:35007 {}] regionserver.HRegionServer(3091): Received CLOSE for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35249 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:50540 deadline: 1733258130972, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b29c245002d9 port=35007 startCode=1733258054368. As of locationSeqNum=12. 2024-12-03T20:34:30,972 INFO [RS:2;b29c245002d9:35007 {}] regionserver.HRegionServer(956): aborting server b29c245002d9,35007,1733258054368 2024-12-03T20:34:30,972 INFO [RS:2;b29c245002d9:35007 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T20:34:30,972 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2fcbcf91e026c1e68ac208aa693d488a, disabling compactions & flushes 2024-12-03T20:34:30,972 INFO [RS:2;b29c245002d9:35007 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;b29c245002d9:35007. 2024-12-03T20:34:30,972 INFO [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:30,972 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:30,972 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35249,1733258054237, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35249,1733258054237, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b29c245002d9 port=35007 startCode=1733258054368. As of locationSeqNum=12. 2024-12-03T20:34:30,972 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. after waiting 0 ms 2024-12-03T20:34:30,972 DEBUG [RS:2;b29c245002d9:35007 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T20:34:30,972 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35249,1733258054237, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b29c245002d9 port=35007 startCode=1733258054368. As of locationSeqNum=12. 2024-12-03T20:34:30,972 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:30,972 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35249,1733258054237, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35007,1733258054368, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=b29c245002d9 port=35007 startCode=1733258054368. As of locationSeqNum=12. 2024-12-03T20:34:30,973 DEBUG [RS:2;b29c245002d9:35007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T20:34:30,973 INFO [RS:2;b29c245002d9:35007 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T20:34:30,973 DEBUG [RS:2;b29c245002d9:35007 {}] regionserver.HRegionServer(1325): Online Regions={2fcbcf91e026c1e68ac208aa693d488a=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.} 2024-12-03T20:34:30,974 DEBUG [RS:2;b29c245002d9:35007 {}] regionserver.HRegionServer(1351): Waiting on 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:30,974 INFO [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:30,974 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2fcbcf91e026c1e68ac208aa693d488a: Waiting for close lock at 1733258070972Running coprocessor pre-close hooks at 1733258070972Disabling compacts and flushes for region at 1733258070972Disabling writes for close at 1733258070972Writing region close event to WAL at 1733258070974 (+2 ms)Running coprocessor post-close hooks at 1733258070974Closed at 1733258070974 2024-12-03T20:34:30,974 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:31,004 INFO [regionserver/b29c245002d9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T20:34:31,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b29c245002d9,35007,1733258054368 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:31,088 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35007,1733258054368, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35007,1733258054368, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b29c245002d9,35007,1733258054368 aborting 2024-12-03T20:34:31,088 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35007,1733258054368, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server b29c245002d9,35007,1733258054368 aborting 2024-12-03T20:34:31,088 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35007,1733258054368, seqNum=12 from cache 2024-12-03T20:34:31,174 INFO [RS:2;b29c245002d9:35007 {}] regionserver.HRegionServer(976): stopping server b29c245002d9,35007,1733258054368; all regions closed. 2024-12-03T20:34:31,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741835_1011 (size=1407) 2024-12-03T20:34:31,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741835_1011 (size=1407) 2024-12-03T20:34:31,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741835_1011 (size=1407) 2024-12-03T20:34:31,180 DEBUG [RS:2;b29c245002d9:35007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T20:34:31,180 INFO [RS:2;b29c245002d9:35007 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T20:34:31,181 INFO [RS:2;b29c245002d9:35007 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T20:34:31,181 INFO [RS:2;b29c245002d9:35007 {}] hbase.ChoreService(370): Chore service for: regionserver/b29c245002d9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T20:34:31,181 INFO [regionserver/b29c245002d9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T20:34:31,182 INFO [RS:2;b29c245002d9:35007 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T20:34:31,182 INFO [RS:2;b29c245002d9:35007 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T20:34:31,182 INFO [RS:2;b29c245002d9:35007 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T20:34:31,182 INFO [RS:2;b29c245002d9:35007 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T20:34:31,182 INFO [RS:2;b29c245002d9:35007 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35007 2024-12-03T20:34:31,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T20:34:31,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35007-0x1019ce700170003, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b29c245002d9,35007,1733258054368 2024-12-03T20:34:31,296 INFO [RS:2;b29c245002d9:35007 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T20:34:31,303 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35007,1733258054368, seqNum=18] 2024-12-03T20:34:31,306 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server b29c245002d9:35007 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: b29c245002d9/172.17.0.3:35007 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:31,306 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35007,1733258054368, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35007,1733258054368, seqNum=18, error=java.net.ConnectException: Call to address=b29c245002d9:35007 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: b29c245002d9/172.17.0.3:35007 2024-12-03T20:34:31,306 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35007,1733258054368, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-03T20:34:31,306 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35007,1733258054368, seqNum=18 from cache 2024-12-03T20:34:31,307 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address b29c245002d9:35007 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: b29c245002d9/172.17.0.3:35007 2024-12-03T20:34:31,312 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b29c245002d9,35007,1733258054368] 2024-12-03T20:34:31,323 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b29c245002d9,35007,1733258054368 already deleted, retry=false 2024-12-03T20:34:31,323 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of b29c245002d9,35007,1733258054368 on b29c245002d9,38205,1733258053478 2024-12-03T20:34:31,327 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure b29c245002d9,35007,1733258054368, splitWal=true, meta=false 2024-12-03T20:34:31,330 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for b29c245002d9,35007,1733258054368 (carryingMeta=false) b29c245002d9,35007,1733258054368/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@622b22be[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-03T20:34:31,330 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure b29c245002d9,35007,1733258054368, splitWal=true, meta=false 2024-12-03T20:34:31,332 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(207): b29c245002d9,35007,1733258054368 had 1 regions 2024-12-03T20:34:31,333 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure b29c245002d9,35007,1733258054368, splitWal=true, meta=false, isMeta: false 2024-12-03T20:34:31,335 DEBUG [PEWorker-5 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35007,1733258054368-splitting 2024-12-03T20:34:31,336 INFO [PEWorker-5 {}] master.SplitWALManager(105): b29c245002d9,35007,1733258054368 WAL count=1, meta=false 2024-12-03T20:34:31,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure b29c245002d9%2C35007%2C1733258054368.1733258055921}] 2024-12-03T20:34:31,344 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=b29c245002d9,35249,1733258054237 2024-12-03T20:34:31,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure b29c245002d9%2C35007%2C1733258054368.1733258055921, worker=b29c245002d9,35249,1733258054237}] 2024-12-03T20:34:31,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35007-0x1019ce700170003, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T20:34:31,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35007-0x1019ce700170003, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T20:34:31,413 INFO [RS:2;b29c245002d9:35007 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T20:34:31,413 INFO [RS:2;b29c245002d9:35007 {}] regionserver.HRegionServer(1031): Exiting; stopping=b29c245002d9,35007,1733258054368; zookeeper connection closed. 2024-12-03T20:34:31,414 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1db44978 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1db44978 2024-12-03T20:34:31,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35249 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-03T20:34:31,522 INFO [RS_LOG_REPLAY_OPS-regionserver/b29c245002d9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35007,1733258054368-splitting/b29c245002d9%2C35007%2C1733258054368.1733258055921, size=1.4 K (1407bytes) 2024-12-03T20:34:31,522 INFO [RS_LOG_REPLAY_OPS-regionserver/b29c245002d9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35007,1733258054368-splitting/b29c245002d9%2C35007%2C1733258054368.1733258055921 2024-12-03T20:34:31,523 INFO [RS_LOG_REPLAY_OPS-regionserver/b29c245002d9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35007,1733258054368-splitting/b29c245002d9%2C35007%2C1733258054368.1733258055921 after 0ms 2024-12-03T20:34:31,525 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b29c245002d9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35007,1733258054368-splitting/b29c245002d9%2C35007%2C1733258054368.1733258055921: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:31,525 INFO [RS_LOG_REPLAY_OPS-regionserver/b29c245002d9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35007,1733258054368-splitting/b29c245002d9%2C35007%2C1733258054368.1733258055921 took 3ms 2024-12-03T20:34:31,533 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b29c245002d9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 2fcbcf91e026c1e68ac208aa693d488a: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-03T20:34:31,533 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b29c245002d9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35007,1733258054368-splitting/b29c245002d9%2C35007%2C1733258054368.1733258055921 so closing down 2024-12-03T20:34:31,534 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b29c245002d9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T20:34:31,534 INFO [RS_LOG_REPLAY_OPS-regionserver/b29c245002d9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T20:34:31,534 INFO [RS_LOG_REPLAY_OPS-regionserver/b29c245002d9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 8 ms; skipped=6; WAL=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35007,1733258054368-splitting/b29c245002d9%2C35007%2C1733258054368.1733258055921, size=1.4 K, length=1407, corrupted=false, cancelled=false 2024-12-03T20:34:31,534 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b29c245002d9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35007,1733258054368-splitting/b29c245002d9%2C35007%2C1733258054368.1733258055921, journal: Splitting hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35007,1733258054368-splitting/b29c245002d9%2C35007%2C1733258054368.1733258055921, size=1.4 K (1407bytes) at 1733258071522Finishing writing output for hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35007,1733258054368-splitting/b29c245002d9%2C35007%2C1733258054368.1733258055921 so closing down at 1733258071533 (+11 ms)3 split writer threads finished at 1733258071534 (+1 ms)Processed 6 edits across 0 Regions in 8 ms; skipped=6; WAL=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35007,1733258054368-splitting/b29c245002d9%2C35007%2C1733258054368.1733258055921, size=1.4 K, length=1407, corrupted=false, cancelled=false at 1733258071534 2024-12-03T20:34:31,534 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b29c245002d9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35007,1733258054368-splitting/b29c245002d9%2C35007%2C1733258054368.1733258055921 2024-12-03T20:34:31,536 DEBUG [RS_LOG_REPLAY_OPS-regionserver/b29c245002d9:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-03T20:34:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38205 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-03T20:34:31,543 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35007,1733258054368-splitting/b29c245002d9%2C35007%2C1733258054368.1733258055921 to hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/oldWALs 2024-12-03T20:34:31,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-03T20:34:31,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure b29c245002d9%2C35007%2C1733258054368.1733258055921, worker=b29c245002d9,35249,1733258054237 in 197 msec 2024-12-03T20:34:31,547 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=b29c245002d9,35249,1733258054237 2024-12-03T20:34:31,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T20:34:31,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure b29c245002d9%2C35007%2C1733258054368.1733258055921, worker=b29c245002d9,35249,1733258054237 in 210 msec 2024-12-03T20:34:31,551 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35007,1733258054368-splitting dir is empty, no logs to split. 2024-12-03T20:34:31,552 INFO [PEWorker-5 {}] master.SplitWALManager(105): b29c245002d9,35007,1733258054368 WAL count=0, meta=false 2024-12-03T20:34:31,552 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if b29c245002d9,35007,1733258054368 WAL splitting is done? wals=0, meta=false 2024-12-03T20:34:31,553 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for b29c245002d9,35007,1733258054368 failed, ignore...File hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/WALs/b29c245002d9,35007,1733258054368-splitting does not exist. 2024-12-03T20:34:31,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, ASSIGN}] 2024-12-03T20:34:31,557 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, ASSIGN 2024-12-03T20:34:31,558 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T20:34:31,616 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35007,1733258054368, seqNum=18] 2024-12-03T20:34:31,617 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to b29c245002d9:35007 this server is in the failed servers list 2024-12-03T20:34:31,618 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35007,1733258054368, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35007,1733258054368, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=b29c245002d9:35007 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b29c245002d9:35007 2024-12-03T20:34:31,618 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35007,1733258054368, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: b29c245002d9:35007 2024-12-03T20:34:31,618 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35007,1733258054368, seqNum=18 from cache 2024-12-03T20:34:31,709 DEBUG [b29c245002d9:38205 {}] balancer.BalancerClusterState(204): Hosts are {b29c245002d9=0} racks are {/default-rack=0} 2024-12-03T20:34:31,709 DEBUG [b29c245002d9:38205 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T20:34:31,709 DEBUG [b29c245002d9:38205 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T20:34:31,709 DEBUG [b29c245002d9:38205 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T20:34:31,709 DEBUG [b29c245002d9:38205 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T20:34:31,709 INFO [b29c245002d9:38205 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T20:34:31,710 INFO [b29c245002d9:38205 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T20:34:31,710 DEBUG [b29c245002d9:38205 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T20:34:31,710 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=2fcbcf91e026c1e68ac208aa693d488a, regionState=OPENING, regionLocation=b29c245002d9,35249,1733258054237 2024-12-03T20:34:31,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, ASSIGN because future has completed 2024-12-03T20:34:31,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2fcbcf91e026c1e68ac208aa693d488a, server=b29c245002d9,35249,1733258054237}] 2024-12-03T20:34:31,876 INFO [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:31,877 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 2fcbcf91e026c1e68ac208aa693d488a, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:31,877 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:31,877 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:31,878 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:31,878 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:31,880 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:31,882 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fcbcf91e026c1e68ac208aa693d488a columnFamilyName cf1 2024-12-03T20:34:31,882 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:31,889 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf1/981cc8d8e7e84a4c9cb43e2bd76e3045 2024-12-03T20:34:31,889 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(327): Store=2fcbcf91e026c1e68ac208aa693d488a/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:31,889 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:31,891 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fcbcf91e026c1e68ac208aa693d488a columnFamilyName cf2 2024-12-03T20:34:31,891 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:31,897 DEBUG [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/cf2/25a1125973414265b47641a4322e8e39 2024-12-03T20:34:31,897 INFO [StoreOpener-2fcbcf91e026c1e68ac208aa693d488a-1 {}] regionserver.HStore(327): Store=2fcbcf91e026c1e68ac208aa693d488a/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:31,897 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:31,898 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:31,900 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:31,901 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:31,901 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:31,901 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T20:34:31,903 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:31,904 INFO [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 2fcbcf91e026c1e68ac208aa693d488a; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69171100, jitterRate=0.030729711055755615}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T20:34:31,904 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:31,904 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 2fcbcf91e026c1e68ac208aa693d488a: Running coprocessor pre-open hook at 1733258071878Writing region info on filesystem at 1733258071878Initializing all the Stores at 1733258071880 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258071880Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258071880Cleaning up temporary data from old regions at 1733258071901 (+21 ms)Running coprocessor post-open hooks at 1733258071904 (+3 ms)Region opened successfully at 1733258071904 2024-12-03T20:34:31,905 INFO [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., pid=17, masterSystemTime=1733258071867 2024-12-03T20:34:31,907 DEBUG [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:31,907 INFO [RS_OPEN_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:31,908 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=2fcbcf91e026c1e68ac208aa693d488a, regionState=OPEN, openSeqNum=18, regionLocation=b29c245002d9,35249,1733258054237 2024-12-03T20:34:31,910 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2fcbcf91e026c1e68ac208aa693d488a, server=b29c245002d9,35249,1733258054237 because future has completed 2024-12-03T20:34:31,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-03T20:34:31,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 2fcbcf91e026c1e68ac208aa693d488a, server=b29c245002d9,35249,1733258054237 in 197 msec 2024-12-03T20:34:31,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-03T20:34:31,916 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server b29c245002d9,35007,1733258054368 after splitting done 2024-12-03T20:34:31,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=2fcbcf91e026c1e68ac208aa693d488a, ASSIGN in 357 msec 2024-12-03T20:34:31,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure b29c245002d9,35007,1733258054368, splitWal=true, meta=false in 592 msec 2024-12-03T20:34:32,127 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a., hostname=b29c245002d9,35249,1733258054237, seqNum=18] 2024-12-03T20:34:32,147 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=410 (was 419), OpenFileDescriptor=1045 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=261 (was 261), ProcessCount=11 (was 11), AvailableMemoryMB=2006 (was 2057) 2024-12-03T20:34:32,149 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1045 is superior to 1024 2024-12-03T20:34:32,160 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=410, OpenFileDescriptor=1045, MaxFileDescriptor=1048576, SystemLoadAverage=261, ProcessCount=11, AvailableMemoryMB=2006 2024-12-03T20:34:32,160 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1045 is superior to 1024 2024-12-03T20:34:32,175 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:32,177 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:32,177 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T20:34:32,180 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-34984863, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/hregion-34984863, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:32,190 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-34984863/hregion-34984863.1733258072180, exclude list is [], retry=0 2024-12-03T20:34:32,193 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:32,193 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:32,193 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:32,195 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-34984863/hregion-34984863.1733258072180 2024-12-03T20:34:32,195 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:36683:36683)] 2024-12-03T20:34:32,196 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6aebb61b719adef7ccf9e8daf8c54b77, NAME => 'testReplayEditsWrittenViaHRegion,,1733258072176.6aebb61b719adef7ccf9e8daf8c54b77.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39361/hbase 2024-12-03T20:34:32,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741893_1071 (size=67) 2024-12-03T20:34:32,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741893_1071 (size=67) 2024-12-03T20:34:32,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741893_1071 (size=67) 2024-12-03T20:34:32,204 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733258072176.6aebb61b719adef7ccf9e8daf8c54b77.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:32,207 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,209 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6aebb61b719adef7ccf9e8daf8c54b77 columnFamilyName a 2024-12-03T20:34:32,209 DEBUG [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:32,209 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] regionserver.HStore(327): Store=6aebb61b719adef7ccf9e8daf8c54b77/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:32,209 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,210 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6aebb61b719adef7ccf9e8daf8c54b77 columnFamilyName b 2024-12-03T20:34:32,210 DEBUG [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:32,211 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] regionserver.HStore(327): Store=6aebb61b719adef7ccf9e8daf8c54b77/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:32,211 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,212 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6aebb61b719adef7ccf9e8daf8c54b77 columnFamilyName c 2024-12-03T20:34:32,212 DEBUG [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:32,212 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] regionserver.HStore(327): Store=6aebb61b719adef7ccf9e8daf8c54b77/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:32,212 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,213 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,213 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,214 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,214 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,214 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T20:34:32,216 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,218 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T20:34:32,219 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6aebb61b719adef7ccf9e8daf8c54b77; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66745739, jitterRate=-0.00541098415851593}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T20:34:32,220 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6aebb61b719adef7ccf9e8daf8c54b77: Writing region info on filesystem at 1733258072204Initializing all the Stores at 1733258072205 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258072205Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258072207 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258072207Cleaning up temporary data from old regions at 1733258072214 (+7 ms)Region opened successfully at 1733258072220 (+6 ms) 2024-12-03T20:34:32,220 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6aebb61b719adef7ccf9e8daf8c54b77, disabling compactions & flushes 2024-12-03T20:34:32,220 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733258072176.6aebb61b719adef7ccf9e8daf8c54b77. 2024-12-03T20:34:32,220 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733258072176.6aebb61b719adef7ccf9e8daf8c54b77. 2024-12-03T20:34:32,220 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733258072176.6aebb61b719adef7ccf9e8daf8c54b77. after waiting 0 ms 2024-12-03T20:34:32,220 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733258072176.6aebb61b719adef7ccf9e8daf8c54b77. 2024-12-03T20:34:32,221 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733258072176.6aebb61b719adef7ccf9e8daf8c54b77. 2024-12-03T20:34:32,221 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6aebb61b719adef7ccf9e8daf8c54b77: Waiting for close lock at 1733258072220Disabling compacts and flushes for region at 1733258072220Disabling writes for close at 1733258072220Writing region close event to WAL at 1733258072221 (+1 ms)Closed at 1733258072221 2024-12-03T20:34:32,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741892_1070 (size=95) 2024-12-03T20:34:32,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741892_1070 (size=95) 2024-12-03T20:34:32,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741892_1070 (size=95) 2024-12-03T20:34:32,226 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T20:34:32,226 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-34984863:(num 1733258072180) 2024-12-03T20:34:32,226 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T20:34:32,228 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:32,245 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175/wal.1733258072229, exclude list is [], retry=0 2024-12-03T20:34:32,248 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:32,248 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:32,249 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:32,250 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175/wal.1733258072229 2024-12-03T20:34:32,250 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:34005:34005)] 2024-12-03T20:34:32,250 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6aebb61b719adef7ccf9e8daf8c54b77, NAME => 'testReplayEditsWrittenViaHRegion,,1733258072176.6aebb61b719adef7ccf9e8daf8c54b77.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:32,251 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733258072176.6aebb61b719adef7ccf9e8daf8c54b77.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:32,251 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,251 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,252 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,253 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6aebb61b719adef7ccf9e8daf8c54b77 columnFamilyName a 2024-12-03T20:34:32,253 DEBUG [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:32,253 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] regionserver.HStore(327): Store=6aebb61b719adef7ccf9e8daf8c54b77/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:32,254 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,254 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6aebb61b719adef7ccf9e8daf8c54b77 columnFamilyName b 2024-12-03T20:34:32,254 DEBUG [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:32,255 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] regionserver.HStore(327): Store=6aebb61b719adef7ccf9e8daf8c54b77/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:32,255 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,256 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6aebb61b719adef7ccf9e8daf8c54b77 columnFamilyName c 2024-12-03T20:34:32,256 DEBUG [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:32,256 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] regionserver.HStore(327): Store=6aebb61b719adef7ccf9e8daf8c54b77/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:32,257 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,258 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,259 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,260 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,260 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,261 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T20:34:32,262 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,263 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6aebb61b719adef7ccf9e8daf8c54b77; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66463244, jitterRate=-0.009620487689971924}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T20:34:32,264 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6aebb61b719adef7ccf9e8daf8c54b77: Writing region info on filesystem at 1733258072251Initializing all the Stores at 1733258072252 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258072252Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258072252Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258072252Cleaning up temporary data from old regions at 1733258072260 (+8 ms)Region opened successfully at 1733258072264 (+4 ms) 2024-12-03T20:34:32,292 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6aebb61b719adef7ccf9e8daf8c54b77 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-03T20:34:32,313 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/.tmp/a/e03e4f9b7c17409695df29ce2a589d5d is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733258072264/Put/seqid=0 2024-12-03T20:34:32,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741895_1073 (size=5958) 2024-12-03T20:34:32,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741895_1073 (size=5958) 2024-12-03T20:34:32,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741895_1073 (size=5958) 2024-12-03T20:34:32,321 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/.tmp/a/e03e4f9b7c17409695df29ce2a589d5d 2024-12-03T20:34:32,342 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/.tmp/b/0261a39b9c924491be609d7571c9389d is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733258072274/Put/seqid=0 2024-12-03T20:34:32,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741896_1074 (size=5958) 2024-12-03T20:34:32,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741896_1074 (size=5958) 2024-12-03T20:34:32,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741896_1074 (size=5958) 2024-12-03T20:34:32,351 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/.tmp/b/0261a39b9c924491be609d7571c9389d 2024-12-03T20:34:32,371 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/.tmp/c/60e0dff365a84e0c934bad108eac6638 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733258072283/Put/seqid=0 2024-12-03T20:34:32,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741897_1075 (size=5958) 2024-12-03T20:34:32,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741897_1075 (size=5958) 2024-12-03T20:34:32,378 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/.tmp/c/60e0dff365a84e0c934bad108eac6638 2024-12-03T20:34:32,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741897_1075 (size=5958) 2024-12-03T20:34:32,385 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/.tmp/a/e03e4f9b7c17409695df29ce2a589d5d as hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/a/e03e4f9b7c17409695df29ce2a589d5d 2024-12-03T20:34:32,390 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/a/e03e4f9b7c17409695df29ce2a589d5d, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T20:34:32,392 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/.tmp/b/0261a39b9c924491be609d7571c9389d as hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/b/0261a39b9c924491be609d7571c9389d 2024-12-03T20:34:32,398 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/b/0261a39b9c924491be609d7571c9389d, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T20:34:32,399 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/.tmp/c/60e0dff365a84e0c934bad108eac6638 as hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/c/60e0dff365a84e0c934bad108eac6638 2024-12-03T20:34:32,406 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/c/60e0dff365a84e0c934bad108eac6638, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T20:34:32,407 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 6aebb61b719adef7ccf9e8daf8c54b77 in 115ms, sequenceid=33, compaction requested=false 2024-12-03T20:34:32,408 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6aebb61b719adef7ccf9e8daf8c54b77: 2024-12-03T20:34:32,408 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6aebb61b719adef7ccf9e8daf8c54b77, disabling compactions & flushes 2024-12-03T20:34:32,408 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733258072176.6aebb61b719adef7ccf9e8daf8c54b77. 2024-12-03T20:34:32,408 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733258072176.6aebb61b719adef7ccf9e8daf8c54b77. 2024-12-03T20:34:32,408 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733258072176.6aebb61b719adef7ccf9e8daf8c54b77. after waiting 0 ms 2024-12-03T20:34:32,408 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733258072176.6aebb61b719adef7ccf9e8daf8c54b77. 2024-12-03T20:34:32,410 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733258072176.6aebb61b719adef7ccf9e8daf8c54b77. 2024-12-03T20:34:32,410 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6aebb61b719adef7ccf9e8daf8c54b77: Waiting for close lock at 1733258072408Disabling compacts and flushes for region at 1733258072408Disabling writes for close at 1733258072408Writing region close event to WAL at 1733258072410 (+2 ms)Closed at 1733258072410 2024-12-03T20:34:32,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741894_1072 (size=3386) 2024-12-03T20:34:32,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741894_1072 (size=3386) 2024-12-03T20:34:32,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741894_1072 (size=3386) 2024-12-03T20:34:32,417 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/b/0261a39b9c924491be609d7571c9389d to hdfs://localhost:39361/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/b/0261a39b9c924491be609d7571c9389d 2024-12-03T20:34:32,434 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39361/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175/wal.1733258072229, size=3.3 K (3386bytes) 2024-12-03T20:34:32,434 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39361/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175/wal.1733258072229 2024-12-03T20:34:32,435 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39361/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175/wal.1733258072229 after 1ms 2024-12-03T20:34:32,437 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175/wal.1733258072229: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:32,437 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39361/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175/wal.1733258072229 took 3ms 2024-12-03T20:34:32,440 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39361/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175/wal.1733258072229 so closing down 2024-12-03T20:34:32,441 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T20:34:32,441 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733258072229.temp 2024-12-03T20:34:32,443 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/recovered.edits/0000000000000000003-wal.1733258072229.temp 2024-12-03T20:34:32,443 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T20:34:32,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741898_1076 (size=2944) 2024-12-03T20:34:32,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741898_1076 (size=2944) 2024-12-03T20:34:32,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741898_1076 (size=2944) 2024-12-03T20:34:32,453 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/recovered.edits/0000000000000000003-wal.1733258072229.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T20:34:32,454 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/recovered.edits/0000000000000000003-wal.1733258072229.temp to hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/recovered.edits/0000000000000000032 2024-12-03T20:34:32,454 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:39361/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175/wal.1733258072229, size=3.3 K, length=3386, corrupted=false, cancelled=false 2024-12-03T20:34:32,454 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39361/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175/wal.1733258072229, journal: Splitting hdfs://localhost:39361/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175/wal.1733258072229, size=3.3 K (3386bytes) at 1733258072434Finishing writing output for hdfs://localhost:39361/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175/wal.1733258072229 so closing down at 1733258072440 (+6 ms)Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/recovered.edits/0000000000000000003-wal.1733258072229.temp at 1733258072443 (+3 ms)3 split writer threads finished at 1733258072443Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/recovered.edits/0000000000000000003-wal.1733258072229.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733258072453 (+10 ms)Rename recovered edits hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/recovered.edits/0000000000000000003-wal.1733258072229.temp to hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/recovered.edits/0000000000000000032 at 1733258072454 (+1 ms)Processed 32 edits across 1 Regions in 16 ms; skipped=2; WAL=hdfs://localhost:39361/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175/wal.1733258072229, size=3.3 K, length=3386, corrupted=false, cancelled=false at 1733258072454 2024-12-03T20:34:32,456 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39361/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175/wal.1733258072229 to hdfs://localhost:39361/hbase/oldWALs/wal.1733258072229 2024-12-03T20:34:32,457 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/recovered.edits/0000000000000000032 2024-12-03T20:34:32,457 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T20:34:32,459 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:32,470 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175/wal.1733258072459, exclude list is [], retry=0 2024-12-03T20:34:32,473 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:32,474 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:32,474 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:32,476 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175/wal.1733258072459 2024-12-03T20:34:32,476 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:36683:36683)] 2024-12-03T20:34:32,476 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6aebb61b719adef7ccf9e8daf8c54b77, NAME => 'testReplayEditsWrittenViaHRegion,,1733258072176.6aebb61b719adef7ccf9e8daf8c54b77.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:32,476 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733258072176.6aebb61b719adef7ccf9e8daf8c54b77.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:32,476 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,476 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,478 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,479 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6aebb61b719adef7ccf9e8daf8c54b77 columnFamilyName a 2024-12-03T20:34:32,479 DEBUG [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:32,486 DEBUG [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/a/e03e4f9b7c17409695df29ce2a589d5d 2024-12-03T20:34:32,486 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] regionserver.HStore(327): Store=6aebb61b719adef7ccf9e8daf8c54b77/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:32,486 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,488 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6aebb61b719adef7ccf9e8daf8c54b77 columnFamilyName b 2024-12-03T20:34:32,488 DEBUG [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:32,489 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] regionserver.HStore(327): Store=6aebb61b719adef7ccf9e8daf8c54b77/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:32,489 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,490 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6aebb61b719adef7ccf9e8daf8c54b77 columnFamilyName c 2024-12-03T20:34:32,490 DEBUG [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:32,495 DEBUG [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/c/60e0dff365a84e0c934bad108eac6638 2024-12-03T20:34:32,496 INFO [StoreOpener-6aebb61b719adef7ccf9e8daf8c54b77-1 {}] regionserver.HStore(327): Store=6aebb61b719adef7ccf9e8daf8c54b77/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:32,496 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,496 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,498 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,498 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/recovered.edits/0000000000000000032 2024-12-03T20:34:32,500 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:32,502 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/recovered.edits/0000000000000000032 2024-12-03T20:34:32,502 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6aebb61b719adef7ccf9e8daf8c54b77 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-03T20:34:32,515 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/.tmp/b/d9d762f1e0eb47f49721206b1065698c is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733258072274/Put/seqid=0 2024-12-03T20:34:32,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741900_1078 (size=5958) 2024-12-03T20:34:32,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741900_1078 (size=5958) 2024-12-03T20:34:32,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741900_1078 (size=5958) 2024-12-03T20:34:32,524 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/.tmp/b/d9d762f1e0eb47f49721206b1065698c 2024-12-03T20:34:32,529 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/.tmp/b/d9d762f1e0eb47f49721206b1065698c as hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/b/d9d762f1e0eb47f49721206b1065698c 2024-12-03T20:34:32,534 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/b/d9d762f1e0eb47f49721206b1065698c, entries=10, sequenceid=32, filesize=5.8 K 2024-12-03T20:34:32,534 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 6aebb61b719adef7ccf9e8daf8c54b77 in 32ms, sequenceid=32, compaction requested=false; wal=null 2024-12-03T20:34:32,535 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/recovered.edits/0000000000000000032 2024-12-03T20:34:32,536 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,536 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,537 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T20:34:32,538 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6aebb61b719adef7ccf9e8daf8c54b77 2024-12-03T20:34:32,540 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/6aebb61b719adef7ccf9e8daf8c54b77/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-03T20:34:32,541 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6aebb61b719adef7ccf9e8daf8c54b77; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60657680, jitterRate=-0.0961301326751709}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T20:34:32,541 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6aebb61b719adef7ccf9e8daf8c54b77: Writing region info on filesystem at 1733258072477Initializing all the Stores at 1733258072478 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258072478Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258072478Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258072478Obtaining lock to block concurrent updates at 1733258072502 (+24 ms)Preparing flush snapshotting stores in 6aebb61b719adef7ccf9e8daf8c54b77 at 1733258072502Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733258072176.6aebb61b719adef7ccf9e8daf8c54b77., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733258072502Flushing stores of testReplayEditsWrittenViaHRegion,,1733258072176.6aebb61b719adef7ccf9e8daf8c54b77. at 1733258072502Flushing 6aebb61b719adef7ccf9e8daf8c54b77/b: creating writer at 1733258072502Flushing 6aebb61b719adef7ccf9e8daf8c54b77/b: appending metadata at 1733258072515 (+13 ms)Flushing 6aebb61b719adef7ccf9e8daf8c54b77/b: closing flushed file at 1733258072515Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e3e21a5: reopening flushed file at 1733258072528 (+13 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 6aebb61b719adef7ccf9e8daf8c54b77 in 32ms, sequenceid=32, compaction requested=false; wal=null at 1733258072534 (+6 ms)Cleaning up temporary data from old regions at 1733258072536 (+2 ms)Region opened successfully at 1733258072541 (+5 ms) 2024-12-03T20:34:32,563 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=419 (was 410) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:57526 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:54550 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:48670 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:44648 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1199604065-172.17.0.3-1733258049083:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1199604065-172.17.0.3-1733258049083:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:56074 [Waiting for operation #38] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1199604065-172.17.0.3-1733258049083:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:54616 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1121 (was 1045) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=261 (was 261), ProcessCount=11 (was 11), AvailableMemoryMB=1997 (was 2006) 2024-12-03T20:34:32,563 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1121 is superior to 1024 2024-12-03T20:34:32,577 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=419, OpenFileDescriptor=1121, MaxFileDescriptor=1048576, SystemLoadAverage=261, ProcessCount=11, AvailableMemoryMB=1990 2024-12-03T20:34:32,577 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1121 is superior to 1024 2024-12-03T20:34:32,590 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:32,593 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:32,594 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T20:34:32,609 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-77142161, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/hregion-77142161, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:32,620 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-77142161/hregion-77142161.1733258072609, exclude list is [], retry=0 2024-12-03T20:34:32,623 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:32,623 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:32,624 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:32,626 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-77142161/hregion-77142161.1733258072609 2024-12-03T20:34:32,627 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:33459:33459)] 2024-12-03T20:34:32,627 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ea4c76f89dddf2fb3b2275eaf812ac99, NAME => 'testReplayEditsAfterAbortingFlush,,1733258072591.ea4c76f89dddf2fb3b2275eaf812ac99.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39361/hbase 2024-12-03T20:34:32,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741902_1080 (size=68) 2024-12-03T20:34:32,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741902_1080 (size=68) 2024-12-03T20:34:32,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741902_1080 (size=68) 2024-12-03T20:34:32,641 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733258072591.ea4c76f89dddf2fb3b2275eaf812ac99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:32,643 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,644 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea4c76f89dddf2fb3b2275eaf812ac99 columnFamilyName a 2024-12-03T20:34:32,644 DEBUG [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:32,645 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] regionserver.HStore(327): Store=ea4c76f89dddf2fb3b2275eaf812ac99/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:32,645 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,646 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea4c76f89dddf2fb3b2275eaf812ac99 columnFamilyName b 2024-12-03T20:34:32,647 DEBUG [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:32,647 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] regionserver.HStore(327): Store=ea4c76f89dddf2fb3b2275eaf812ac99/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:32,647 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,648 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea4c76f89dddf2fb3b2275eaf812ac99 columnFamilyName c 2024-12-03T20:34:32,649 DEBUG [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:32,649 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] regionserver.HStore(327): Store=ea4c76f89dddf2fb3b2275eaf812ac99/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:32,649 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,650 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,650 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,652 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,652 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,652 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T20:34:32,654 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,656 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T20:34:32,657 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ea4c76f89dddf2fb3b2275eaf812ac99; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62444660, jitterRate=-0.06950205564498901}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T20:34:32,658 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ea4c76f89dddf2fb3b2275eaf812ac99: Writing region info on filesystem at 1733258072641Initializing all the Stores at 1733258072642 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258072642Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258072643 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258072643Cleaning up temporary data from old regions at 1733258072652 (+9 ms)Region opened successfully at 1733258072658 (+6 ms) 2024-12-03T20:34:32,658 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ea4c76f89dddf2fb3b2275eaf812ac99, disabling compactions & flushes 2024-12-03T20:34:32,658 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733258072591.ea4c76f89dddf2fb3b2275eaf812ac99. 2024-12-03T20:34:32,658 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733258072591.ea4c76f89dddf2fb3b2275eaf812ac99. 2024-12-03T20:34:32,658 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733258072591.ea4c76f89dddf2fb3b2275eaf812ac99. after waiting 0 ms 2024-12-03T20:34:32,658 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733258072591.ea4c76f89dddf2fb3b2275eaf812ac99. 2024-12-03T20:34:32,659 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733258072591.ea4c76f89dddf2fb3b2275eaf812ac99. 2024-12-03T20:34:32,659 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ea4c76f89dddf2fb3b2275eaf812ac99: Waiting for close lock at 1733258072658Disabling compacts and flushes for region at 1733258072658Disabling writes for close at 1733258072658Writing region close event to WAL at 1733258072659 (+1 ms)Closed at 1733258072659 2024-12-03T20:34:32,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741901_1079 (size=95) 2024-12-03T20:34:32,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741901_1079 (size=95) 2024-12-03T20:34:32,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741901_1079 (size=95) 2024-12-03T20:34:32,668 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T20:34:32,668 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-77142161:(num 1733258072609) 2024-12-03T20:34:32,669 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T20:34:32,670 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:32,689 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589/wal.1733258072671, exclude list is [], retry=0 2024-12-03T20:34:32,692 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:32,692 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:32,693 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:32,695 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589/wal.1733258072671 2024-12-03T20:34:32,695 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:36683:36683)] 2024-12-03T20:34:32,765 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ea4c76f89dddf2fb3b2275eaf812ac99, NAME => 'testReplayEditsAfterAbortingFlush,,1733258072591.ea4c76f89dddf2fb3b2275eaf812ac99.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:32,768 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,768 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733258072591.ea4c76f89dddf2fb3b2275eaf812ac99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:32,768 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,768 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,770 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,771 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea4c76f89dddf2fb3b2275eaf812ac99 columnFamilyName a 2024-12-03T20:34:32,772 DEBUG [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:32,772 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] regionserver.HStore(327): Store=ea4c76f89dddf2fb3b2275eaf812ac99/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:32,773 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,774 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea4c76f89dddf2fb3b2275eaf812ac99 columnFamilyName b 2024-12-03T20:34:32,774 DEBUG [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:32,774 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] regionserver.HStore(327): Store=ea4c76f89dddf2fb3b2275eaf812ac99/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:32,775 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,776 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea4c76f89dddf2fb3b2275eaf812ac99 columnFamilyName c 2024-12-03T20:34:32,776 DEBUG [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:32,776 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] regionserver.HStore(327): Store=ea4c76f89dddf2fb3b2275eaf812ac99/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:32,777 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,777 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,778 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,779 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,779 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,780 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T20:34:32,781 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,782 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ea4c76f89dddf2fb3b2275eaf812ac99; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68871689, jitterRate=0.026268139481544495}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T20:34:32,782 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:32,783 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ea4c76f89dddf2fb3b2275eaf812ac99: Running coprocessor pre-open hook at 1733258072768Writing region info on filesystem at 1733258072768Initializing all the Stores at 1733258072769 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258072769Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258072769Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258072769Cleaning up temporary data from old regions at 1733258072779 (+10 ms)Running coprocessor post-open hooks at 1733258072782 (+3 ms)Region opened successfully at 1733258072783 (+1 ms) 2024-12-03T20:34:32,797 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ea4c76f89dddf2fb3b2275eaf812ac99 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-03T20:34:32,799 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ea4c76f89dddf2fb3b2275eaf812ac99/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:33,787 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-03T20:34:33,787 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-03T20:34:33,791 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-03T20:34:33,791 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-03T20:34:33,793 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-03T20:34:33,793 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-03T20:34:33,795 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-03T20:34:33,795 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-03T20:34:33,797 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-03T20:34:33,797 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-03T20:34:33,800 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ea4c76f89dddf2fb3b2275eaf812ac99/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:34,647 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T20:34:34,801 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ea4c76f89dddf2fb3b2275eaf812ac99/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:35,802 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ea4c76f89dddf2fb3b2275eaf812ac99/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:36,803 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ea4c76f89dddf2fb3b2275eaf812ac99/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:37,805 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ea4c76f89dddf2fb3b2275eaf812ac99/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:38,806 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ea4c76f89dddf2fb3b2275eaf812ac99/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:39,808 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ea4c76f89dddf2fb3b2275eaf812ac99/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:40,809 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ea4c76f89dddf2fb3b2275eaf812ac99/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:41,810 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ea4c76f89dddf2fb3b2275eaf812ac99/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:41,812 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ea4c76f89dddf2fb3b2275eaf812ac99: 2024-12-03T20:34:41,812 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733258072591.ea4c76f89dddf2fb3b2275eaf812ac99. 2024-12-03T20:34:41,828 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ea4c76f89dddf2fb3b2275eaf812ac99: 2024-12-03T20:34:41,828 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-03T20:34:41,828 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ea4c76f89dddf2fb3b2275eaf812ac99, disabling compactions & flushes 2024-12-03T20:34:41,828 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733258072591.ea4c76f89dddf2fb3b2275eaf812ac99. 2024-12-03T20:34:41,828 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733258072591.ea4c76f89dddf2fb3b2275eaf812ac99. 2024-12-03T20:34:41,828 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733258072591.ea4c76f89dddf2fb3b2275eaf812ac99. after waiting 0 ms 2024-12-03T20:34:41,828 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733258072591.ea4c76f89dddf2fb3b2275eaf812ac99. 2024-12-03T20:34:41,828 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733258072591.ea4c76f89dddf2fb3b2275eaf812ac99. 2024-12-03T20:34:41,828 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733258072591.ea4c76f89dddf2fb3b2275eaf812ac99. 2024-12-03T20:34:41,828 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ea4c76f89dddf2fb3b2275eaf812ac99: Waiting for close lock at 1733258081828Running coprocessor pre-close hooks at 1733258081828Disabling compacts and flushes for region at 1733258081828Disabling writes for close at 1733258081828Writing region close event to WAL at 1733258081828Running coprocessor post-close hooks at 1733258081828Closed at 1733258081828 2024-12-03T20:34:41,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741903_1081 (size=2691) 2024-12-03T20:34:41,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741903_1081 (size=2691) 2024-12-03T20:34:41,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741903_1081 (size=2691) 2024-12-03T20:34:41,844 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39361/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589/wal.1733258072671, size=2.6 K (2691bytes) 2024-12-03T20:34:41,844 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39361/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589/wal.1733258072671 2024-12-03T20:34:41,844 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39361/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589/wal.1733258072671 after 0ms 2024-12-03T20:34:41,846 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589/wal.1733258072671: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:41,847 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39361/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589/wal.1733258072671 took 3ms 2024-12-03T20:34:41,849 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39361/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589/wal.1733258072671 so closing down 2024-12-03T20:34:41,849 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T20:34:41,850 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733258072671.temp 2024-12-03T20:34:41,850 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/recovered.edits/0000000000000000004-wal.1733258072671.temp 2024-12-03T20:34:41,851 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T20:34:41,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741904_1082 (size=2094) 2024-12-03T20:34:41,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741904_1082 (size=2094) 2024-12-03T20:34:41,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741904_1082 (size=2094) 2024-12-03T20:34:41,860 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/recovered.edits/0000000000000000004-wal.1733258072671.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-03T20:34:41,861 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/recovered.edits/0000000000000000004-wal.1733258072671.temp to hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/recovered.edits/0000000000000000026 2024-12-03T20:34:41,862 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 15 ms; skipped=3; WAL=hdfs://localhost:39361/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589/wal.1733258072671, size=2.6 K, length=2691, corrupted=false, cancelled=false 2024-12-03T20:34:41,862 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39361/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589/wal.1733258072671, journal: Splitting hdfs://localhost:39361/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589/wal.1733258072671, size=2.6 K (2691bytes) at 1733258081844Finishing writing output for hdfs://localhost:39361/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589/wal.1733258072671 so closing down at 1733258081849 (+5 ms)Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/recovered.edits/0000000000000000004-wal.1733258072671.temp at 1733258081850 (+1 ms)3 split writer threads finished at 1733258081851 (+1 ms)Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/recovered.edits/0000000000000000004-wal.1733258072671.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733258081860 (+9 ms)Rename recovered edits hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/recovered.edits/0000000000000000004-wal.1733258072671.temp to hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/recovered.edits/0000000000000000026 at 1733258081861 (+1 ms)Processed 23 edits across 1 Regions in 15 ms; skipped=3; WAL=hdfs://localhost:39361/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589/wal.1733258072671, size=2.6 K, length=2691, corrupted=false, cancelled=false at 1733258081862 (+1 ms) 2024-12-03T20:34:41,864 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39361/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589/wal.1733258072671 to hdfs://localhost:39361/hbase/oldWALs/wal.1733258072671 2024-12-03T20:34:41,864 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/recovered.edits/0000000000000000026 2024-12-03T20:34:41,865 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T20:34:41,866 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:41,878 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589/wal.1733258081866, exclude list is [], retry=0 2024-12-03T20:34:41,880 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:41,880 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:41,881 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:41,882 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589/wal.1733258081866 2024-12-03T20:34:41,882 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:33459:33459)] 2024-12-03T20:34:41,882 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ea4c76f89dddf2fb3b2275eaf812ac99, NAME => 'testReplayEditsAfterAbortingFlush,,1733258072591.ea4c76f89dddf2fb3b2275eaf812ac99.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:41,883 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:41,883 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733258072591.ea4c76f89dddf2fb3b2275eaf812ac99.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:41,883 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:41,883 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:41,885 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:41,886 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea4c76f89dddf2fb3b2275eaf812ac99 columnFamilyName a 2024-12-03T20:34:41,886 DEBUG [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:41,886 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] regionserver.HStore(327): Store=ea4c76f89dddf2fb3b2275eaf812ac99/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:41,886 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:41,887 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea4c76f89dddf2fb3b2275eaf812ac99 columnFamilyName b 2024-12-03T20:34:41,887 DEBUG [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:41,888 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] regionserver.HStore(327): Store=ea4c76f89dddf2fb3b2275eaf812ac99/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:41,888 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:41,889 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea4c76f89dddf2fb3b2275eaf812ac99 columnFamilyName c 2024-12-03T20:34:41,889 DEBUG [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:41,890 INFO [StoreOpener-ea4c76f89dddf2fb3b2275eaf812ac99-1 {}] regionserver.HStore(327): Store=ea4c76f89dddf2fb3b2275eaf812ac99/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:41,890 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:41,890 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:41,892 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:41,892 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/recovered.edits/0000000000000000026 2024-12-03T20:34:41,894 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:41,895 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/recovered.edits/0000000000000000026 2024-12-03T20:34:41,896 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ea4c76f89dddf2fb3b2275eaf812ac99 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-03T20:34:41,908 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/.tmp/a/688fa98389a14f73bd01145417988bff is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733258081819/Put/seqid=0 2024-12-03T20:34:41,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741906_1084 (size=5523) 2024-12-03T20:34:41,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741906_1084 (size=5523) 2024-12-03T20:34:41,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741906_1084 (size=5523) 2024-12-03T20:34:41,914 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/.tmp/a/688fa98389a14f73bd01145417988bff 2024-12-03T20:34:41,936 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/.tmp/b/51da4dd527694249bdf8b137ec6cdf4c is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733258081812/Put/seqid=0 2024-12-03T20:34:41,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741907_1085 (size=5524) 2024-12-03T20:34:41,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741907_1085 (size=5524) 2024-12-03T20:34:41,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741907_1085 (size=5524) 2024-12-03T20:34:41,943 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/.tmp/b/51da4dd527694249bdf8b137ec6cdf4c 2024-12-03T20:34:41,962 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/.tmp/c/720c461e4d1e4a12bf72ae2948fb06a6 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733258081816/Put/seqid=0 2024-12-03T20:34:41,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741908_1086 (size=5457) 2024-12-03T20:34:41,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741908_1086 (size=5457) 2024-12-03T20:34:41,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741908_1086 (size=5457) 2024-12-03T20:34:41,969 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/.tmp/c/720c461e4d1e4a12bf72ae2948fb06a6 2024-12-03T20:34:41,976 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/.tmp/a/688fa98389a14f73bd01145417988bff as hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/a/688fa98389a14f73bd01145417988bff 2024-12-03T20:34:41,981 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/a/688fa98389a14f73bd01145417988bff, entries=7, sequenceid=26, filesize=5.4 K 2024-12-03T20:34:41,983 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/.tmp/b/51da4dd527694249bdf8b137ec6cdf4c as hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/b/51da4dd527694249bdf8b137ec6cdf4c 2024-12-03T20:34:41,988 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/b/51da4dd527694249bdf8b137ec6cdf4c, entries=7, sequenceid=26, filesize=5.4 K 2024-12-03T20:34:41,989 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/.tmp/c/720c461e4d1e4a12bf72ae2948fb06a6 as hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/c/720c461e4d1e4a12bf72ae2948fb06a6 2024-12-03T20:34:41,994 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/c/720c461e4d1e4a12bf72ae2948fb06a6, entries=6, sequenceid=26, filesize=5.3 K 2024-12-03T20:34:41,994 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for ea4c76f89dddf2fb3b2275eaf812ac99 in 99ms, sequenceid=26, compaction requested=false; wal=null 2024-12-03T20:34:41,995 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/recovered.edits/0000000000000000026 2024-12-03T20:34:41,996 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:41,996 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:41,997 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T20:34:41,998 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:42,000 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testReplayEditsAfterAbortingFlush/ea4c76f89dddf2fb3b2275eaf812ac99/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-03T20:34:42,001 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ea4c76f89dddf2fb3b2275eaf812ac99; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71846363, jitterRate=0.07059423625469208}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T20:34:42,002 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ea4c76f89dddf2fb3b2275eaf812ac99 2024-12-03T20:34:42,002 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ea4c76f89dddf2fb3b2275eaf812ac99: Running coprocessor pre-open hook at 1733258081883Writing region info on filesystem at 1733258081883Initializing all the Stores at 1733258081884 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258081884Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258081884Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258081884Obtaining lock to block concurrent updates at 1733258081896 (+12 ms)Preparing flush snapshotting stores in ea4c76f89dddf2fb3b2275eaf812ac99 at 1733258081896Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733258072591.ea4c76f89dddf2fb3b2275eaf812ac99., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733258081896Flushing stores of testReplayEditsAfterAbortingFlush,,1733258072591.ea4c76f89dddf2fb3b2275eaf812ac99. at 1733258081896Flushing ea4c76f89dddf2fb3b2275eaf812ac99/a: creating writer at 1733258081896Flushing ea4c76f89dddf2fb3b2275eaf812ac99/a: appending metadata at 1733258081907 (+11 ms)Flushing ea4c76f89dddf2fb3b2275eaf812ac99/a: closing flushed file at 1733258081907Flushing ea4c76f89dddf2fb3b2275eaf812ac99/b: creating writer at 1733258081918 (+11 ms)Flushing ea4c76f89dddf2fb3b2275eaf812ac99/b: appending metadata at 1733258081935 (+17 ms)Flushing ea4c76f89dddf2fb3b2275eaf812ac99/b: closing flushed file at 1733258081936 (+1 ms)Flushing ea4c76f89dddf2fb3b2275eaf812ac99/c: creating writer at 1733258081948 (+12 ms)Flushing ea4c76f89dddf2fb3b2275eaf812ac99/c: appending metadata at 1733258081961 (+13 ms)Flushing ea4c76f89dddf2fb3b2275eaf812ac99/c: closing flushed file at 1733258081962 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a8796e1: reopening flushed file at 1733258081975 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3dd552fe: reopening flushed file at 1733258081982 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61bd7c31: reopening flushed file at 1733258081988 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for ea4c76f89dddf2fb3b2275eaf812ac99 in 99ms, sequenceid=26, compaction requested=false; wal=null at 1733258081994 (+6 ms)Cleaning up temporary data from old regions at 1733258081996 (+2 ms)Running coprocessor post-open hooks at 1733258082002 (+6 ms)Region opened successfully at 1733258082002 2024-12-03T20:34:42,019 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=420 (was 419) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:34796 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:34812 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1199604065-172.17.0.3-1733258049083:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1199604065-172.17.0.3-1733258049083:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1199604065-172.17.0.3-1733258049083:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:46888 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:54668 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:46530 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:46492 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1179 (was 1121) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=220 (was 261), ProcessCount=11 (was 11), AvailableMemoryMB=1984 (was 1990) 2024-12-03T20:34:42,019 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1179 is superior to 1024 2024-12-03T20:34:42,030 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=420, OpenFileDescriptor=1179, MaxFileDescriptor=1048576, SystemLoadAverage=220, ProcessCount=11, AvailableMemoryMB=1983 2024-12-03T20:34:42,030 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1179 is superior to 1024 2024-12-03T20:34:42,042 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:42,044 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:42,045 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T20:34:42,047 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-12840624, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/hregion-12840624, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:42,057 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-12840624/hregion-12840624.1733258082047, exclude list is [], retry=0 2024-12-03T20:34:42,060 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:42,060 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:42,060 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:42,062 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-12840624/hregion-12840624.1733258082047 2024-12-03T20:34:42,062 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:34005:34005)] 2024-12-03T20:34:42,062 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c1e0b8d4d0e30f370b59eca888610444, NAME => 'testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39361/hbase 2024-12-03T20:34:42,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741910_1088 (size=61) 2024-12-03T20:34:42,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741910_1088 (size=61) 2024-12-03T20:34:42,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741910_1088 (size=61) 2024-12-03T20:34:42,073 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:42,074 INFO [StoreOpener-c1e0b8d4d0e30f370b59eca888610444-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,075 INFO [StoreOpener-c1e0b8d4d0e30f370b59eca888610444-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1e0b8d4d0e30f370b59eca888610444 columnFamilyName a 2024-12-03T20:34:42,075 DEBUG [StoreOpener-c1e0b8d4d0e30f370b59eca888610444-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:42,076 INFO [StoreOpener-c1e0b8d4d0e30f370b59eca888610444-1 {}] regionserver.HStore(327): Store=c1e0b8d4d0e30f370b59eca888610444/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:42,076 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,076 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,077 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,077 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,077 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,079 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,081 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T20:34:42,081 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c1e0b8d4d0e30f370b59eca888610444; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69014064, jitterRate=0.028389692306518555}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T20:34:42,082 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c1e0b8d4d0e30f370b59eca888610444: Writing region info on filesystem at 1733258082073Initializing all the Stores at 1733258082074 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258082074Cleaning up temporary data from old regions at 1733258082077 (+3 ms)Region opened successfully at 1733258082081 (+4 ms) 2024-12-03T20:34:42,082 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c1e0b8d4d0e30f370b59eca888610444, disabling compactions & flushes 2024-12-03T20:34:42,082 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444. 2024-12-03T20:34:42,082 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444. 2024-12-03T20:34:42,082 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444. after waiting 0 ms 2024-12-03T20:34:42,082 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444. 2024-12-03T20:34:42,082 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444. 2024-12-03T20:34:42,082 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c1e0b8d4d0e30f370b59eca888610444: Waiting for close lock at 1733258082082Disabling compacts and flushes for region at 1733258082082Disabling writes for close at 1733258082082Writing region close event to WAL at 1733258082082Closed at 1733258082082 2024-12-03T20:34:42,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741909_1087 (size=95) 2024-12-03T20:34:42,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741909_1087 (size=95) 2024-12-03T20:34:42,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741909_1087 (size=95) 2024-12-03T20:34:42,087 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T20:34:42,087 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-12840624:(num 1733258082047) 2024-12-03T20:34:42,087 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T20:34:42,089 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:42,102 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041/wal.1733258082089, exclude list is [], retry=0 2024-12-03T20:34:42,104 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:42,105 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:42,105 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:42,106 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041/wal.1733258082089 2024-12-03T20:34:42,107 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:34005:34005)] 2024-12-03T20:34:42,107 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c1e0b8d4d0e30f370b59eca888610444, NAME => 'testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:42,107 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:42,107 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,107 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,109 INFO [StoreOpener-c1e0b8d4d0e30f370b59eca888610444-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,109 INFO [StoreOpener-c1e0b8d4d0e30f370b59eca888610444-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1e0b8d4d0e30f370b59eca888610444 columnFamilyName a 2024-12-03T20:34:42,109 DEBUG [StoreOpener-c1e0b8d4d0e30f370b59eca888610444-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:42,110 INFO [StoreOpener-c1e0b8d4d0e30f370b59eca888610444-1 {}] regionserver.HStore(327): Store=c1e0b8d4d0e30f370b59eca888610444/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:42,110 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,111 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,112 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,112 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,112 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,113 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,114 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c1e0b8d4d0e30f370b59eca888610444; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64002557, jitterRate=-0.04628758132457733}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T20:34:42,115 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c1e0b8d4d0e30f370b59eca888610444: Writing region info on filesystem at 1733258082107Initializing all the Stores at 1733258082108 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258082108Cleaning up temporary data from old regions at 1733258082112 (+4 ms)Region opened successfully at 1733258082115 (+3 ms) 2024-12-03T20:34:42,122 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c1e0b8d4d0e30f370b59eca888610444, disabling compactions & flushes 2024-12-03T20:34:42,122 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444. 2024-12-03T20:34:42,122 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444. 2024-12-03T20:34:42,122 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444. after waiting 0 ms 2024-12-03T20:34:42,122 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444. 2024-12-03T20:34:42,123 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444. 2024-12-03T20:34:42,123 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444. 2024-12-03T20:34:42,123 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c1e0b8d4d0e30f370b59eca888610444: Waiting for close lock at 1733258082122Disabling compacts and flushes for region at 1733258082122Disabling writes for close at 1733258082122Writing region close event to WAL at 1733258082123 (+1 ms)Closed at 1733258082123 2024-12-03T20:34:42,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741911_1089 (size=1050) 2024-12-03T20:34:42,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741911_1089 (size=1050) 2024-12-03T20:34:42,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741911_1089 (size=1050) 2024-12-03T20:34:42,139 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39361/hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041/wal.1733258082089, size=1.0 K (1050bytes) 2024-12-03T20:34:42,139 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39361/hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041/wal.1733258082089 2024-12-03T20:34:42,140 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39361/hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041/wal.1733258082089 after 0ms 2024-12-03T20:34:42,141 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041/wal.1733258082089: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:42,141 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39361/hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041/wal.1733258082089 took 2ms 2024-12-03T20:34:42,143 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39361/hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041/wal.1733258082089 so closing down 2024-12-03T20:34:42,143 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T20:34:42,144 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733258082089.temp 2024-12-03T20:34:42,145 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/recovered.edits/0000000000000000003-wal.1733258082089.temp 2024-12-03T20:34:42,146 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T20:34:42,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741912_1090 (size=1050) 2024-12-03T20:34:42,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741912_1090 (size=1050) 2024-12-03T20:34:42,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741912_1090 (size=1050) 2024-12-03T20:34:42,152 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/recovered.edits/0000000000000000003-wal.1733258082089.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-03T20:34:42,154 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/recovered.edits/0000000000000000003-wal.1733258082089.temp to hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/recovered.edits/0000000000000000012 2024-12-03T20:34:42,154 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:39361/hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041/wal.1733258082089, size=1.0 K, length=1050, corrupted=false, cancelled=false 2024-12-03T20:34:42,154 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39361/hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041/wal.1733258082089, journal: Splitting hdfs://localhost:39361/hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041/wal.1733258082089, size=1.0 K (1050bytes) at 1733258082139Finishing writing output for hdfs://localhost:39361/hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041/wal.1733258082089 so closing down at 1733258082143 (+4 ms)Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/recovered.edits/0000000000000000003-wal.1733258082089.temp at 1733258082145 (+2 ms)3 split writer threads finished at 1733258082146 (+1 ms)Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/recovered.edits/0000000000000000003-wal.1733258082089.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733258082152 (+6 ms)Rename recovered edits hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/recovered.edits/0000000000000000003-wal.1733258082089.temp to hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/recovered.edits/0000000000000000012 at 1733258082154 (+2 ms)Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:39361/hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041/wal.1733258082089, size=1.0 K, length=1050, corrupted=false, cancelled=false at 1733258082154 2024-12-03T20:34:42,156 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39361/hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041/wal.1733258082089 to hdfs://localhost:39361/hbase/oldWALs/wal.1733258082089 2024-12-03T20:34:42,157 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/recovered.edits/0000000000000000012 2024-12-03T20:34:42,160 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:42,476 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T20:34:42,477 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:42,488 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041/wal.1733258082478, exclude list is [], retry=0 2024-12-03T20:34:42,490 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:42,490 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:42,491 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:42,492 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041/wal.1733258082478 2024-12-03T20:34:42,492 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:36683:36683)] 2024-12-03T20:34:42,492 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c1e0b8d4d0e30f370b59eca888610444, NAME => 'testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:42,492 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:42,493 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,493 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,495 INFO [StoreOpener-c1e0b8d4d0e30f370b59eca888610444-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,497 INFO [StoreOpener-c1e0b8d4d0e30f370b59eca888610444-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1e0b8d4d0e30f370b59eca888610444 columnFamilyName a 2024-12-03T20:34:42,497 DEBUG [StoreOpener-c1e0b8d4d0e30f370b59eca888610444-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:42,497 INFO [StoreOpener-c1e0b8d4d0e30f370b59eca888610444-1 {}] regionserver.HStore(327): Store=c1e0b8d4d0e30f370b59eca888610444/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:42,498 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,498 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,499 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,500 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/recovered.edits/0000000000000000012 2024-12-03T20:34:42,502 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:42,503 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/recovered.edits/0000000000000000012 2024-12-03T20:34:42,503 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c1e0b8d4d0e30f370b59eca888610444 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-03T20:34:42,519 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/.tmp/a/8306092e09b54e8ba2cef0051581acfa is 79, key is testDatalossWhenInputError/a:x0/1733258082115/Put/seqid=0 2024-12-03T20:34:42,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741914_1092 (size=5808) 2024-12-03T20:34:42,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741914_1092 (size=5808) 2024-12-03T20:34:42,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741914_1092 (size=5808) 2024-12-03T20:34:42,526 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/.tmp/a/8306092e09b54e8ba2cef0051581acfa 2024-12-03T20:34:42,535 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/.tmp/a/8306092e09b54e8ba2cef0051581acfa as hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/a/8306092e09b54e8ba2cef0051581acfa 2024-12-03T20:34:42,543 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/a/8306092e09b54e8ba2cef0051581acfa, entries=10, sequenceid=12, filesize=5.7 K 2024-12-03T20:34:42,543 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for c1e0b8d4d0e30f370b59eca888610444 in 40ms, sequenceid=12, compaction requested=false; wal=null 2024-12-03T20:34:42,544 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/recovered.edits/0000000000000000012 2024-12-03T20:34:42,545 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,545 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,547 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,549 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-03T20:34:42,550 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c1e0b8d4d0e30f370b59eca888610444; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74169863, jitterRate=0.1052170842885971}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T20:34:42,551 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c1e0b8d4d0e30f370b59eca888610444: Writing region info on filesystem at 1733258082493Initializing all the Stores at 1733258082495 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258082495Obtaining lock to block concurrent updates at 1733258082503 (+8 ms)Preparing flush snapshotting stores in c1e0b8d4d0e30f370b59eca888610444 at 1733258082503Finished memstore snapshotting testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733258082503Flushing stores of testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444. at 1733258082503Flushing c1e0b8d4d0e30f370b59eca888610444/a: creating writer at 1733258082503Flushing c1e0b8d4d0e30f370b59eca888610444/a: appending metadata at 1733258082519 (+16 ms)Flushing c1e0b8d4d0e30f370b59eca888610444/a: closing flushed file at 1733258082519Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@632720dd: reopening flushed file at 1733258082533 (+14 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for c1e0b8d4d0e30f370b59eca888610444 in 40ms, sequenceid=12, compaction requested=false; wal=null at 1733258082543 (+10 ms)Cleaning up temporary data from old regions at 1733258082545 (+2 ms)Region opened successfully at 1733258082551 (+6 ms) 2024-12-03T20:34:42,553 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c1e0b8d4d0e30f370b59eca888610444, NAME => 'testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:42,553 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733258082043.c1e0b8d4d0e30f370b59eca888610444.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:42,553 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,553 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,555 INFO [StoreOpener-c1e0b8d4d0e30f370b59eca888610444-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,555 INFO [StoreOpener-c1e0b8d4d0e30f370b59eca888610444-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c1e0b8d4d0e30f370b59eca888610444 columnFamilyName a 2024-12-03T20:34:42,555 DEBUG [StoreOpener-c1e0b8d4d0e30f370b59eca888610444-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:42,560 DEBUG [StoreOpener-c1e0b8d4d0e30f370b59eca888610444-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/a/8306092e09b54e8ba2cef0051581acfa 2024-12-03T20:34:42,561 INFO [StoreOpener-c1e0b8d4d0e30f370b59eca888610444-1 {}] regionserver.HStore(327): Store=c1e0b8d4d0e30f370b59eca888610444/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:42,561 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,561 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,563 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,563 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,563 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,565 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c1e0b8d4d0e30f370b59eca888610444 2024-12-03T20:34:42,567 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testDatalossWhenInputError/c1e0b8d4d0e30f370b59eca888610444/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-03T20:34:42,568 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c1e0b8d4d0e30f370b59eca888610444; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65656549, jitterRate=-0.02164117991924286}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T20:34:42,569 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c1e0b8d4d0e30f370b59eca888610444: Writing region info on filesystem at 1733258082553Initializing all the Stores at 1733258082554 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258082554Cleaning up temporary data from old regions at 1733258082563 (+9 ms)Region opened successfully at 1733258082569 (+6 ms) 2024-12-03T20:34:42,584 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=430 (was 420) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:34812 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:46594 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1199604065-172.17.0.3-1733258049083:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:46974 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:34882 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:54668 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:46530 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1199604065-172.17.0.3-1733258049083:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1199604065-172.17.0.3-1733258049083:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1261 (was 1179) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=220 (was 220), ProcessCount=11 (was 11), AvailableMemoryMB=1977 (was 1983) 2024-12-03T20:34:42,584 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1261 is superior to 1024 2024-12-03T20:34:42,596 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=430, OpenFileDescriptor=1261, MaxFileDescriptor=1048576, SystemLoadAverage=220, ProcessCount=11, AvailableMemoryMB=1976 2024-12-03T20:34:42,596 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1261 is superior to 1024 2024-12-03T20:34:42,610 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:42,611 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:42,612 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T20:34:42,614 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-16886733, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/hregion-16886733, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:42,625 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-16886733/hregion-16886733.1733258082614, exclude list is [], retry=0 2024-12-03T20:34:42,627 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:42,627 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:42,628 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:42,629 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-16886733/hregion-16886733.1733258082614 2024-12-03T20:34:42,630 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:36683:36683)] 2024-12-03T20:34:42,630 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 92deccad882021ab99ad15b8d3c89503, NAME => 'testCompactedBulkLoadedFiles,,1733258082610.92deccad882021ab99ad15b8d3c89503.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39361/hbase 2024-12-03T20:34:42,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741916_1094 (size=63) 2024-12-03T20:34:42,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741916_1094 (size=63) 2024-12-03T20:34:42,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741916_1094 (size=63) 2024-12-03T20:34:42,641 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733258082610.92deccad882021ab99ad15b8d3c89503.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:42,642 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:42,643 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92deccad882021ab99ad15b8d3c89503 columnFamilyName a 2024-12-03T20:34:42,644 DEBUG [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:42,644 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.HStore(327): Store=92deccad882021ab99ad15b8d3c89503/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:42,644 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:42,645 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92deccad882021ab99ad15b8d3c89503 columnFamilyName b 2024-12-03T20:34:42,645 DEBUG [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:42,645 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.HStore(327): Store=92deccad882021ab99ad15b8d3c89503/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:42,646 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:42,646 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92deccad882021ab99ad15b8d3c89503 columnFamilyName c 2024-12-03T20:34:42,647 DEBUG [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:42,647 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.HStore(327): Store=92deccad882021ab99ad15b8d3c89503/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:42,647 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:42,648 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:42,648 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:42,649 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:42,649 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:42,649 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T20:34:42,650 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:42,652 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T20:34:42,652 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 92deccad882021ab99ad15b8d3c89503; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63131791, jitterRate=-0.05926300585269928}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T20:34:42,653 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 92deccad882021ab99ad15b8d3c89503: Writing region info on filesystem at 1733258082641Initializing all the Stores at 1733258082642 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258082642Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258082642Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258082642Cleaning up temporary data from old regions at 1733258082649 (+7 ms)Region opened successfully at 1733258082653 (+4 ms) 2024-12-03T20:34:42,653 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 92deccad882021ab99ad15b8d3c89503, disabling compactions & flushes 2024-12-03T20:34:42,653 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733258082610.92deccad882021ab99ad15b8d3c89503. 2024-12-03T20:34:42,653 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733258082610.92deccad882021ab99ad15b8d3c89503. 2024-12-03T20:34:42,653 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733258082610.92deccad882021ab99ad15b8d3c89503. after waiting 0 ms 2024-12-03T20:34:42,653 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733258082610.92deccad882021ab99ad15b8d3c89503. 2024-12-03T20:34:42,653 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733258082610.92deccad882021ab99ad15b8d3c89503. 2024-12-03T20:34:42,653 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 92deccad882021ab99ad15b8d3c89503: Waiting for close lock at 1733258082653Disabling compacts and flushes for region at 1733258082653Disabling writes for close at 1733258082653Writing region close event to WAL at 1733258082653Closed at 1733258082653 2024-12-03T20:34:42,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741915_1093 (size=95) 2024-12-03T20:34:42,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741915_1093 (size=95) 2024-12-03T20:34:42,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741915_1093 (size=95) 2024-12-03T20:34:42,658 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T20:34:42,658 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-16886733:(num 1733258082614) 2024-12-03T20:34:42,658 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T20:34:42,659 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:42,664 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T20:34:42,670 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258082660, exclude list is [], retry=0 2024-12-03T20:34:42,673 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:42,673 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:42,673 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:42,674 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258082660 2024-12-03T20:34:42,675 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:34005:34005)] 2024-12-03T20:34:42,675 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 92deccad882021ab99ad15b8d3c89503, NAME => 'testCompactedBulkLoadedFiles,,1733258082610.92deccad882021ab99ad15b8d3c89503.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:42,675 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733258082610.92deccad882021ab99ad15b8d3c89503.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:42,675 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:42,675 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:42,677 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:42,678 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92deccad882021ab99ad15b8d3c89503 columnFamilyName a 2024-12-03T20:34:42,678 DEBUG [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:42,678 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.HStore(327): Store=92deccad882021ab99ad15b8d3c89503/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:42,679 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:42,680 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92deccad882021ab99ad15b8d3c89503 columnFamilyName b 2024-12-03T20:34:42,680 DEBUG [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:42,680 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.HStore(327): Store=92deccad882021ab99ad15b8d3c89503/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:42,680 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:42,681 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92deccad882021ab99ad15b8d3c89503 columnFamilyName c 2024-12-03T20:34:42,681 DEBUG [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:42,682 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.HStore(327): Store=92deccad882021ab99ad15b8d3c89503/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:42,682 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:42,683 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:42,684 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:42,685 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:42,685 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:42,685 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T20:34:42,687 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:42,688 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 92deccad882021ab99ad15b8d3c89503; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64875503, jitterRate=-0.033279672265052795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T20:34:42,689 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 92deccad882021ab99ad15b8d3c89503: Writing region info on filesystem at 1733258082675Initializing all the Stores at 1733258082676 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258082676Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258082677 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258082677Cleaning up temporary data from old regions at 1733258082685 (+8 ms)Region opened successfully at 1733258082689 (+4 ms) 2024-12-03T20:34:42,693 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733258082693/Put/seqid=0 2024-12-03T20:34:42,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741918_1096 (size=4875) 2024-12-03T20:34:42,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741918_1096 (size=4875) 2024-12-03T20:34:42,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741918_1096 (size=4875) 2024-12-03T20:34:42,713 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733258082713/Put/seqid=0 2024-12-03T20:34:42,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741919_1097 (size=4875) 2024-12-03T20:34:42,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741919_1097 (size=4875) 2024-12-03T20:34:42,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741919_1097 (size=4875) 2024-12-03T20:34:42,723 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733258082722/Put/seqid=0 2024-12-03T20:34:42,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741920_1098 (size=4875) 2024-12-03T20:34:42,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741920_1098 (size=4875) 2024-12-03T20:34:42,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741920_1098 (size=4875) 2024-12-03T20:34:42,734 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:39361/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 92deccad882021ab99ad15b8d3c89503/a 2024-12-03T20:34:42,738 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-03T20:34:42,738 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T20:34:42,738 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:39361/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 92deccad882021ab99ad15b8d3c89503/a 2024-12-03T20:34:42,742 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-03T20:34:42,742 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T20:34:42,742 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:39361/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 92deccad882021ab99ad15b8d3c89503/a 2024-12-03T20:34:42,746 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-03T20:34:42,746 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T20:34:42,746 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 92deccad882021ab99ad15b8d3c89503 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-03T20:34:42,760 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/.tmp/a/eedffba485b34e80b6cee70100e03c3b is 55, key is testCompactedBulkLoadedFiles/a:a/1733258082689/Put/seqid=0 2024-12-03T20:34:42,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741921_1099 (size=5107) 2024-12-03T20:34:42,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741921_1099 (size=5107) 2024-12-03T20:34:42,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741921_1099 (size=5107) 2024-12-03T20:34:42,768 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/.tmp/a/eedffba485b34e80b6cee70100e03c3b 2024-12-03T20:34:42,773 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/.tmp/a/eedffba485b34e80b6cee70100e03c3b as hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/eedffba485b34e80b6cee70100e03c3b 2024-12-03T20:34:42,778 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/eedffba485b34e80b6cee70100e03c3b, entries=1, sequenceid=4, filesize=5.0 K 2024-12-03T20:34:42,779 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 92deccad882021ab99ad15b8d3c89503 in 32ms, sequenceid=4, compaction requested=false 2024-12-03T20:34:42,779 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 92deccad882021ab99ad15b8d3c89503: 2024-12-03T20:34:42,780 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/707617a4e9b84f189301991a442c1b82_SeqId_4_ 2024-12-03T20:34:42,781 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/bf3fbcdcff79404386a701d9e09c655d_SeqId_4_ 2024-12-03T20:34:42,782 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/1eb25cbfecc84b3e93f22b873dfa1921_SeqId_4_ 2024-12-03T20:34:42,782 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:39361/hbase/testCompactedBulkLoadedFiles/hfile0 into 92deccad882021ab99ad15b8d3c89503/a as hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/707617a4e9b84f189301991a442c1b82_SeqId_4_ - updating store file list. 2024-12-03T20:34:42,786 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 707617a4e9b84f189301991a442c1b82_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T20:34:42,786 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/707617a4e9b84f189301991a442c1b82_SeqId_4_ into 92deccad882021ab99ad15b8d3c89503/a 2024-12-03T20:34:42,786 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:39361/hbase/testCompactedBulkLoadedFiles/hfile0 into 92deccad882021ab99ad15b8d3c89503/a (new location: hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/707617a4e9b84f189301991a442c1b82_SeqId_4_) 2024-12-03T20:34:42,787 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:39361/hbase/testCompactedBulkLoadedFiles/hfile1 into 92deccad882021ab99ad15b8d3c89503/a as hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/bf3fbcdcff79404386a701d9e09c655d_SeqId_4_ - updating store file list. 2024-12-03T20:34:42,792 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for bf3fbcdcff79404386a701d9e09c655d_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T20:34:42,792 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/bf3fbcdcff79404386a701d9e09c655d_SeqId_4_ into 92deccad882021ab99ad15b8d3c89503/a 2024-12-03T20:34:42,792 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:39361/hbase/testCompactedBulkLoadedFiles/hfile1 into 92deccad882021ab99ad15b8d3c89503/a (new location: hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/bf3fbcdcff79404386a701d9e09c655d_SeqId_4_) 2024-12-03T20:34:42,794 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:39361/hbase/testCompactedBulkLoadedFiles/hfile2 into 92deccad882021ab99ad15b8d3c89503/a as hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/1eb25cbfecc84b3e93f22b873dfa1921_SeqId_4_ - updating store file list. 2024-12-03T20:34:42,798 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1eb25cbfecc84b3e93f22b873dfa1921_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T20:34:42,798 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/1eb25cbfecc84b3e93f22b873dfa1921_SeqId_4_ into 92deccad882021ab99ad15b8d3c89503/a 2024-12-03T20:34:42,798 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:39361/hbase/testCompactedBulkLoadedFiles/hfile2 into 92deccad882021ab99ad15b8d3c89503/a (new location: hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/1eb25cbfecc84b3e93f22b873dfa1921_SeqId_4_) 2024-12-03T20:34:42,806 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T20:34:42,806 DEBUG [Time-limited test {}] regionserver.HStore(1541): 92deccad882021ab99ad15b8d3c89503/a is initiating major compaction (all files) 2024-12-03T20:34:42,806 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 92deccad882021ab99ad15b8d3c89503/a in testCompactedBulkLoadedFiles,,1733258082610.92deccad882021ab99ad15b8d3c89503. 2024-12-03T20:34:42,807 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/eedffba485b34e80b6cee70100e03c3b, hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/707617a4e9b84f189301991a442c1b82_SeqId_4_, hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/bf3fbcdcff79404386a701d9e09c655d_SeqId_4_, hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/1eb25cbfecc84b3e93f22b873dfa1921_SeqId_4_] into tmpdir=hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/.tmp, totalSize=19.3 K 2024-12-03T20:34:42,807 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting eedffba485b34e80b6cee70100e03c3b, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733258082689 2024-12-03T20:34:42,808 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 707617a4e9b84f189301991a442c1b82_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T20:34:42,808 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting bf3fbcdcff79404386a701d9e09c655d_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T20:34:42,809 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1eb25cbfecc84b3e93f22b873dfa1921_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T20:34:42,823 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/.tmp/a/b40ff8a85ea142168cad78020e32e241 is 55, key is testCompactedBulkLoadedFiles/a:a/1733258082689/Put/seqid=0 2024-12-03T20:34:42,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741922_1100 (size=6154) 2024-12-03T20:34:42,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741922_1100 (size=6154) 2024-12-03T20:34:42,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741922_1100 (size=6154) 2024-12-03T20:34:42,835 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/.tmp/a/b40ff8a85ea142168cad78020e32e241 as hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/b40ff8a85ea142168cad78020e32e241 2024-12-03T20:34:42,844 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 92deccad882021ab99ad15b8d3c89503/a of 92deccad882021ab99ad15b8d3c89503 into b40ff8a85ea142168cad78020e32e241(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T20:34:42,844 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 92deccad882021ab99ad15b8d3c89503: 2024-12-03T20:34:42,844 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-03T20:34:42,844 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-03T20:34:42,879 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39361/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258082660, size=0 (0bytes) 2024-12-03T20:34:42,879 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:39361/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258082660 might be still open, length is 0 2024-12-03T20:34:42,879 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39361/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258082660 2024-12-03T20:34:42,880 WARN [IPC Server handler 0 on default port 39361 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258082660 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-03T20:34:42,880 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39361/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258082660 after 1ms 2024-12-03T20:34:43,787 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-03T20:34:43,787 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-03T20:34:43,790 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-03T20:34:43,790 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-03T20:34:45,565 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:34912 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:33321:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34912 dst: /127.0.0.1:33321 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33321 remote=/127.0.0.1:34912]. Total timeout mills is 60000, 57278 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:45,566 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:46628 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:36623:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46628 dst: /127.0.0.1:36623 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:45,566 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:47012 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:46119:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47012 dst: /127.0.0.1:46119 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:45,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741917_1101 (size=1174) 2024-12-03T20:34:45,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741917_1101 (size=1174) 2024-12-03T20:34:46,881 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39361/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258082660 after 4002ms 2024-12-03T20:34:46,884 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258082660: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:46,885 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39361/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258082660 took 4006ms 2024-12-03T20:34:46,887 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:39361/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258082660; continuing. 2024-12-03T20:34:46,887 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39361/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258082660 so closing down 2024-12-03T20:34:46,887 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T20:34:46,889 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733258082660.temp 2024-12-03T20:34:46,891 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/recovered.edits/0000000000000000003-wal.1733258082660.temp 2024-12-03T20:34:46,891 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T20:34:46,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741923_1102 (size=548) 2024-12-03T20:34:46,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741923_1102 (size=548) 2024-12-03T20:34:46,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741923_1102 (size=548) 2024-12-03T20:34:46,901 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/recovered.edits/0000000000000000003-wal.1733258082660.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T20:34:46,903 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/recovered.edits/0000000000000000003-wal.1733258082660.temp to hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/recovered.edits/0000000000000000008 2024-12-03T20:34:46,903 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 18 ms; skipped=3; WAL=hdfs://localhost:39361/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258082660, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T20:34:46,903 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39361/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258082660, journal: Splitting hdfs://localhost:39361/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258082660, size=0 (0bytes) at 1733258082879Finishing writing output for hdfs://localhost:39361/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258082660 so closing down at 1733258086887 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/recovered.edits/0000000000000000003-wal.1733258082660.temp at 1733258086891 (+4 ms)3 split writer threads finished at 1733258086891Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/recovered.edits/0000000000000000003-wal.1733258082660.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733258086901 (+10 ms)Rename recovered edits hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/recovered.edits/0000000000000000003-wal.1733258082660.temp to hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/recovered.edits/0000000000000000008 at 1733258086903 (+2 ms)Processed 5 edits across 1 Regions in 18 ms; skipped=3; WAL=hdfs://localhost:39361/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258082660, size=0, length=0, corrupted=false, cancelled=false at 1733258086903 2024-12-03T20:34:46,905 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39361/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258082660 to hdfs://localhost:39361/hbase/oldWALs/wal.1733258082660 2024-12-03T20:34:46,906 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/recovered.edits/0000000000000000008 2024-12-03T20:34:46,906 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T20:34:46,908 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:46,921 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258086908, exclude list is [], retry=0 2024-12-03T20:34:46,923 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:46,923 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:46,924 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:46,925 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258086908 2024-12-03T20:34:46,925 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:34005:34005)] 2024-12-03T20:34:46,925 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 92deccad882021ab99ad15b8d3c89503, NAME => 'testCompactedBulkLoadedFiles,,1733258082610.92deccad882021ab99ad15b8d3c89503.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:46,926 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733258082610.92deccad882021ab99ad15b8d3c89503.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:46,926 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:46,926 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:46,927 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:46,927 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92deccad882021ab99ad15b8d3c89503 columnFamilyName a 2024-12-03T20:34:46,928 DEBUG [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:46,934 DEBUG [StoreFileOpener-92deccad882021ab99ad15b8d3c89503-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1eb25cbfecc84b3e93f22b873dfa1921_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T20:34:46,934 DEBUG [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/1eb25cbfecc84b3e93f22b873dfa1921_SeqId_4_ 2024-12-03T20:34:46,938 DEBUG [StoreFileOpener-92deccad882021ab99ad15b8d3c89503-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 707617a4e9b84f189301991a442c1b82_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T20:34:46,938 DEBUG [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/707617a4e9b84f189301991a442c1b82_SeqId_4_ 2024-12-03T20:34:46,942 DEBUG [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/b40ff8a85ea142168cad78020e32e241 2024-12-03T20:34:46,945 DEBUG [StoreFileOpener-92deccad882021ab99ad15b8d3c89503-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for bf3fbcdcff79404386a701d9e09c655d_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T20:34:46,946 DEBUG [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/bf3fbcdcff79404386a701d9e09c655d_SeqId_4_ 2024-12-03T20:34:46,949 DEBUG [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/eedffba485b34e80b6cee70100e03c3b 2024-12-03T20:34:46,949 WARN [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/1eb25cbfecc84b3e93f22b873dfa1921_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5bdd00e5 2024-12-03T20:34:46,950 WARN [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/707617a4e9b84f189301991a442c1b82_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5bdd00e5 2024-12-03T20:34:46,950 WARN [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/bf3fbcdcff79404386a701d9e09c655d_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5bdd00e5 2024-12-03T20:34:46,950 WARN [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/eedffba485b34e80b6cee70100e03c3b from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@5bdd00e5 2024-12-03T20:34:46,950 DEBUG [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/1eb25cbfecc84b3e93f22b873dfa1921_SeqId_4_, hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/707617a4e9b84f189301991a442c1b82_SeqId_4_, hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/bf3fbcdcff79404386a701d9e09c655d_SeqId_4_, hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/eedffba485b34e80b6cee70100e03c3b] to archive 2024-12-03T20:34:46,950 DEBUG [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T20:34:46,952 DEBUG [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/1eb25cbfecc84b3e93f22b873dfa1921_SeqId_4_ to hdfs://localhost:39361/hbase/archive/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/1eb25cbfecc84b3e93f22b873dfa1921_SeqId_4_ 2024-12-03T20:34:46,953 DEBUG [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/707617a4e9b84f189301991a442c1b82_SeqId_4_ to hdfs://localhost:39361/hbase/archive/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/707617a4e9b84f189301991a442c1b82_SeqId_4_ 2024-12-03T20:34:46,954 DEBUG [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/bf3fbcdcff79404386a701d9e09c655d_SeqId_4_ to hdfs://localhost:39361/hbase/archive/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/bf3fbcdcff79404386a701d9e09c655d_SeqId_4_ 2024-12-03T20:34:46,955 DEBUG [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/eedffba485b34e80b6cee70100e03c3b to hdfs://localhost:39361/hbase/archive/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/a/eedffba485b34e80b6cee70100e03c3b 2024-12-03T20:34:46,955 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.HStore(327): Store=92deccad882021ab99ad15b8d3c89503/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:46,955 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:46,956 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92deccad882021ab99ad15b8d3c89503 columnFamilyName b 2024-12-03T20:34:46,956 DEBUG [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:46,956 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.HStore(327): Store=92deccad882021ab99ad15b8d3c89503/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:46,956 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:46,957 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92deccad882021ab99ad15b8d3c89503 columnFamilyName c 2024-12-03T20:34:46,957 DEBUG [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:46,957 INFO [StoreOpener-92deccad882021ab99ad15b8d3c89503-1 {}] regionserver.HStore(327): Store=92deccad882021ab99ad15b8d3c89503/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:46,957 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:46,958 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:46,959 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:46,960 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/recovered.edits/0000000000000000008 2024-12-03T20:34:46,961 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:46,963 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 92deccad882021ab99ad15b8d3c89503 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "92deccad882021ab99ad15b8d3c89503" family_name: "a" compaction_input: "eedffba485b34e80b6cee70100e03c3b" compaction_input: "707617a4e9b84f189301991a442c1b82_SeqId_4_" compaction_input: "bf3fbcdcff79404386a701d9e09c655d_SeqId_4_" compaction_input: "1eb25cbfecc84b3e93f22b873dfa1921_SeqId_4_" compaction_output: "b40ff8a85ea142168cad78020e32e241" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733258082610.92deccad882021ab99ad15b8d3c89503." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-03T20:34:46,963 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-03T20:34:46,963 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/recovered.edits/0000000000000000008 2024-12-03T20:34:46,964 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/recovered.edits/0000000000000000008 2024-12-03T20:34:46,965 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:46,965 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:46,965 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T20:34:46,967 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 92deccad882021ab99ad15b8d3c89503 2024-12-03T20:34:46,968 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testCompactedBulkLoadedFiles/92deccad882021ab99ad15b8d3c89503/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-03T20:34:46,969 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 92deccad882021ab99ad15b8d3c89503; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69661685, jitterRate=0.03803999722003937}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T20:34:46,969 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 92deccad882021ab99ad15b8d3c89503: Writing region info on filesystem at 1733258086926Initializing all the Stores at 1733258086926Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258086927 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258086927Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258086927Cleaning up temporary data from old regions at 1733258086965 (+38 ms)Region opened successfully at 1733258086969 (+4 ms) 2024-12-03T20:34:46,971 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 92deccad882021ab99ad15b8d3c89503, disabling compactions & flushes 2024-12-03T20:34:46,971 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733258082610.92deccad882021ab99ad15b8d3c89503. 2024-12-03T20:34:46,971 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733258082610.92deccad882021ab99ad15b8d3c89503. 2024-12-03T20:34:46,972 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733258082610.92deccad882021ab99ad15b8d3c89503. after waiting 0 ms 2024-12-03T20:34:46,972 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733258082610.92deccad882021ab99ad15b8d3c89503. 2024-12-03T20:34:46,972 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733258082610.92deccad882021ab99ad15b8d3c89503. 2024-12-03T20:34:46,972 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 92deccad882021ab99ad15b8d3c89503: Waiting for close lock at 1733258086971Disabling compacts and flushes for region at 1733258086971Disabling writes for close at 1733258086972 (+1 ms)Writing region close event to WAL at 1733258086972Closed at 1733258086972 2024-12-03T20:34:46,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741924_1103 (size=95) 2024-12-03T20:34:46,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741924_1103 (size=95) 2024-12-03T20:34:46,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741924_1103 (size=95) 2024-12-03T20:34:46,976 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T20:34:46,976 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733258086908) 2024-12-03T20:34:46,987 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=444 (was 430) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-979745026_22 at /127.0.0.1:46692 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:39361 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-979745026_22 at /127.0.0.1:34988 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:39361 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-979745026_22 at /127.0.0.1:47060 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1342 (was 1261) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=203 (was 220), ProcessCount=11 (was 11), AvailableMemoryMB=1967 (was 1976) 2024-12-03T20:34:46,987 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1342 is superior to 1024 2024-12-03T20:34:46,998 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=444, OpenFileDescriptor=1342, MaxFileDescriptor=1048576, SystemLoadAverage=203, ProcessCount=11, AvailableMemoryMB=1967 2024-12-03T20:34:46,998 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1342 is superior to 1024 2024-12-03T20:34:47,011 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:47,012 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T20:34:47,013 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T20:34:47,015 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-49577623, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/hregion-49577623, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:47,026 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-49577623/hregion-49577623.1733258087015, exclude list is [], retry=0 2024-12-03T20:34:47,028 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:47,028 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:47,028 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:47,030 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-49577623/hregion-49577623.1733258087015 2024-12-03T20:34:47,030 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:36683:36683)] 2024-12-03T20:34:47,030 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f3fb69afe3beda4eed21cce50a04d55d, NAME => 'testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39361/hbase 2024-12-03T20:34:47,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741926_1105 (size=67) 2024-12-03T20:34:47,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741926_1105 (size=67) 2024-12-03T20:34:47,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741926_1105 (size=67) 2024-12-03T20:34:47,038 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:47,039 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,040 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3fb69afe3beda4eed21cce50a04d55d columnFamilyName a 2024-12-03T20:34:47,040 DEBUG [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:47,041 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(327): Store=f3fb69afe3beda4eed21cce50a04d55d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:47,041 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,042 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3fb69afe3beda4eed21cce50a04d55d columnFamilyName b 2024-12-03T20:34:47,042 DEBUG [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:47,043 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(327): Store=f3fb69afe3beda4eed21cce50a04d55d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:47,043 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,044 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3fb69afe3beda4eed21cce50a04d55d columnFamilyName c 2024-12-03T20:34:47,044 DEBUG [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:47,044 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(327): Store=f3fb69afe3beda4eed21cce50a04d55d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:47,044 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,045 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,045 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,046 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,046 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,047 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T20:34:47,048 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,051 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T20:34:47,051 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f3fb69afe3beda4eed21cce50a04d55d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58765821, jitterRate=-0.12432102859020233}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T20:34:47,051 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f3fb69afe3beda4eed21cce50a04d55d: Writing region info on filesystem at 1733258087038Initializing all the Stores at 1733258087039 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258087039Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258087039Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258087039Cleaning up temporary data from old regions at 1733258087046 (+7 ms)Region opened successfully at 1733258087051 (+5 ms) 2024-12-03T20:34:47,051 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f3fb69afe3beda4eed21cce50a04d55d, disabling compactions & flushes 2024-12-03T20:34:47,051 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d. 2024-12-03T20:34:47,051 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d. 2024-12-03T20:34:47,051 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d. after waiting 0 ms 2024-12-03T20:34:47,051 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d. 2024-12-03T20:34:47,052 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d. 2024-12-03T20:34:47,052 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f3fb69afe3beda4eed21cce50a04d55d: Waiting for close lock at 1733258087051Disabling compacts and flushes for region at 1733258087051Disabling writes for close at 1733258087051Writing region close event to WAL at 1733258087052 (+1 ms)Closed at 1733258087052 2024-12-03T20:34:47,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741925_1104 (size=95) 2024-12-03T20:34:47,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741925_1104 (size=95) 2024-12-03T20:34:47,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741925_1104 (size=95) 2024-12-03T20:34:47,057 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T20:34:47,057 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-49577623:(num 1733258087015) 2024-12-03T20:34:47,057 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T20:34:47,059 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:47,072 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087059, exclude list is [], retry=0 2024-12-03T20:34:47,075 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:47,075 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:47,075 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:47,077 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087059 2024-12-03T20:34:47,077 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:34005:34005)] 2024-12-03T20:34:47,077 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f3fb69afe3beda4eed21cce50a04d55d, NAME => 'testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:47,077 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:47,077 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,077 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,079 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,079 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3fb69afe3beda4eed21cce50a04d55d columnFamilyName a 2024-12-03T20:34:47,079 DEBUG [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:47,080 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(327): Store=f3fb69afe3beda4eed21cce50a04d55d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:47,080 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,081 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3fb69afe3beda4eed21cce50a04d55d columnFamilyName b 2024-12-03T20:34:47,081 DEBUG [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:47,081 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(327): Store=f3fb69afe3beda4eed21cce50a04d55d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:47,081 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,082 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3fb69afe3beda4eed21cce50a04d55d columnFamilyName c 2024-12-03T20:34:47,082 DEBUG [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:47,083 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(327): Store=f3fb69afe3beda4eed21cce50a04d55d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:47,083 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,083 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,084 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,085 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,085 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,086 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T20:34:47,087 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,087 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f3fb69afe3beda4eed21cce50a04d55d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59709783, jitterRate=-0.11025489866733551}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T20:34:47,087 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f3fb69afe3beda4eed21cce50a04d55d: Writing region info on filesystem at 1733258087077Initializing all the Stores at 1733258087078 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258087078Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258087078Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258087078Cleaning up temporary data from old regions at 1733258087085 (+7 ms)Region opened successfully at 1733258087087 (+2 ms) 2024-12-03T20:34:47,095 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f3fb69afe3beda4eed21cce50a04d55d 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-03T20:34:47,109 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/.tmp/a/97007ba3fbda4db5af9ab5f04dc70371 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733258087087/Put/seqid=0 2024-12-03T20:34:47,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741928_1107 (size=5958) 2024-12-03T20:34:47,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741928_1107 (size=5958) 2024-12-03T20:34:47,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741928_1107 (size=5958) 2024-12-03T20:34:47,115 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/.tmp/a/97007ba3fbda4db5af9ab5f04dc70371 2024-12-03T20:34:47,120 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/.tmp/a/97007ba3fbda4db5af9ab5f04dc70371 as hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/a/97007ba3fbda4db5af9ab5f04dc70371 2024-12-03T20:34:47,124 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/a/97007ba3fbda4db5af9ab5f04dc70371, entries=10, sequenceid=13, filesize=5.8 K 2024-12-03T20:34:47,125 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for f3fb69afe3beda4eed21cce50a04d55d in 30ms, sequenceid=13, compaction requested=false 2024-12-03T20:34:47,125 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f3fb69afe3beda4eed21cce50a04d55d: 2024-12-03T20:34:47,141 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f3fb69afe3beda4eed21cce50a04d55d, disabling compactions & flushes 2024-12-03T20:34:47,141 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d. 2024-12-03T20:34:47,141 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d. 2024-12-03T20:34:47,141 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d. after waiting 0 ms 2024-12-03T20:34:47,141 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d. 2024-12-03T20:34:47,142 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d. 2024-12-03T20:34:47,142 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d. 2024-12-03T20:34:47,142 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f3fb69afe3beda4eed21cce50a04d55d: Waiting for close lock at 1733258087141Disabling compacts and flushes for region at 1733258087141Disabling writes for close at 1733258087141Writing region close event to WAL at 1733258087142 (+1 ms)Closed at 1733258087142 2024-12-03T20:34:47,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741927_1106 (size=3346) 2024-12-03T20:34:47,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741927_1106 (size=3346) 2024-12-03T20:34:47,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741927_1106 (size=3346) 2024-12-03T20:34:47,158 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087059, size=3.3 K (3346bytes) 2024-12-03T20:34:47,158 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087059 2024-12-03T20:34:47,158 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087059 after 0ms 2024-12-03T20:34:47,160 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087059: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:47,160 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087059 took 2ms 2024-12-03T20:34:47,162 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087059 so closing down 2024-12-03T20:34:47,162 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T20:34:47,163 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733258087059.temp 2024-12-03T20:34:47,164 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000003-wal.1733258087059.temp 2024-12-03T20:34:47,164 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T20:34:47,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741929_1108 (size=2944) 2024-12-03T20:34:47,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741929_1108 (size=2944) 2024-12-03T20:34:47,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741929_1108 (size=2944) 2024-12-03T20:34:47,170 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000003-wal.1733258087059.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T20:34:47,172 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000003-wal.1733258087059.temp to hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000035 2024-12-03T20:34:47,172 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 11 ms; skipped=2; WAL=hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087059, size=3.3 K, length=3346, corrupted=false, cancelled=false 2024-12-03T20:34:47,172 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087059, journal: Splitting hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087059, size=3.3 K (3346bytes) at 1733258087158Finishing writing output for hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087059 so closing down at 1733258087162 (+4 ms)Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000003-wal.1733258087059.temp at 1733258087164 (+2 ms)3 split writer threads finished at 1733258087164Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000003-wal.1733258087059.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733258087170 (+6 ms)Rename recovered edits hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000003-wal.1733258087059.temp to hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000035 at 1733258087172 (+2 ms)Processed 32 edits across 1 Regions in 11 ms; skipped=2; WAL=hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087059, size=3.3 K, length=3346, corrupted=false, cancelled=false at 1733258087172 2024-12-03T20:34:47,173 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087059 to hdfs://localhost:39361/hbase/oldWALs/wal.1733258087059 2024-12-03T20:34:47,174 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000035 2024-12-03T20:34:47,174 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T20:34:47,176 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:47,190 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087176, exclude list is [], retry=0 2024-12-03T20:34:47,192 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:47,192 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:47,192 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:47,193 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087176 2024-12-03T20:34:47,194 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:36683:36683),(127.0.0.1/127.0.0.1:34005:34005)] 2024-12-03T20:34:47,194 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f3fb69afe3beda4eed21cce50a04d55d, NAME => 'testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d.', STARTKEY => '', ENDKEY => ''} 2024-12-03T20:34:47,194 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:47,194 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,194 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,196 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,196 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3fb69afe3beda4eed21cce50a04d55d columnFamilyName a 2024-12-03T20:34:47,196 DEBUG [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:47,201 DEBUG [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/a/97007ba3fbda4db5af9ab5f04dc70371 2024-12-03T20:34:47,201 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(327): Store=f3fb69afe3beda4eed21cce50a04d55d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:47,201 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,202 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3fb69afe3beda4eed21cce50a04d55d columnFamilyName b 2024-12-03T20:34:47,202 DEBUG [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:47,203 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(327): Store=f3fb69afe3beda4eed21cce50a04d55d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:47,203 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,203 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3fb69afe3beda4eed21cce50a04d55d columnFamilyName c 2024-12-03T20:34:47,204 DEBUG [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:47,204 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(327): Store=f3fb69afe3beda4eed21cce50a04d55d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:47,204 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,205 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,206 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,207 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000035 2024-12-03T20:34:47,209 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:47,210 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000035 2024-12-03T20:34:47,210 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f3fb69afe3beda4eed21cce50a04d55d 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-03T20:34:47,226 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/.tmp/b/096c197683d54b6588ecf185b7f01a14 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733258087125/Put/seqid=0 2024-12-03T20:34:47,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741931_1110 (size=5958) 2024-12-03T20:34:47,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741931_1110 (size=5958) 2024-12-03T20:34:47,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741931_1110 (size=5958) 2024-12-03T20:34:47,232 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/.tmp/b/096c197683d54b6588ecf185b7f01a14 2024-12-03T20:34:47,249 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/.tmp/c/4ca8f46b79bf43f5820f4849feb1a3a6 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733258087131/Put/seqid=0 2024-12-03T20:34:47,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741932_1111 (size=5958) 2024-12-03T20:34:47,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741932_1111 (size=5958) 2024-12-03T20:34:47,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741932_1111 (size=5958) 2024-12-03T20:34:47,255 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/.tmp/c/4ca8f46b79bf43f5820f4849feb1a3a6 2024-12-03T20:34:47,260 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/.tmp/b/096c197683d54b6588ecf185b7f01a14 as hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/b/096c197683d54b6588ecf185b7f01a14 2024-12-03T20:34:47,265 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/b/096c197683d54b6588ecf185b7f01a14, entries=10, sequenceid=35, filesize=5.8 K 2024-12-03T20:34:47,266 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/.tmp/c/4ca8f46b79bf43f5820f4849feb1a3a6 as hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/c/4ca8f46b79bf43f5820f4849feb1a3a6 2024-12-03T20:34:47,272 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/c/4ca8f46b79bf43f5820f4849feb1a3a6, entries=10, sequenceid=35, filesize=5.8 K 2024-12-03T20:34:47,272 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for f3fb69afe3beda4eed21cce50a04d55d in 62ms, sequenceid=35, compaction requested=false; wal=null 2024-12-03T20:34:47,273 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000035 2024-12-03T20:34:47,274 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,274 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,275 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T20:34:47,277 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:47,279 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-03T20:34:47,280 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f3fb69afe3beda4eed21cce50a04d55d; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74108886, jitterRate=0.10430845618247986}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T20:34:47,280 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f3fb69afe3beda4eed21cce50a04d55d: Writing region info on filesystem at 1733258087194Initializing all the Stores at 1733258087195 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258087195Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258087195Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258087195Obtaining lock to block concurrent updates at 1733258087210 (+15 ms)Preparing flush snapshotting stores in f3fb69afe3beda4eed21cce50a04d55d at 1733258087210Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733258087210Flushing stores of testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d. at 1733258087210Flushing f3fb69afe3beda4eed21cce50a04d55d/b: creating writer at 1733258087210Flushing f3fb69afe3beda4eed21cce50a04d55d/b: appending metadata at 1733258087225 (+15 ms)Flushing f3fb69afe3beda4eed21cce50a04d55d/b: closing flushed file at 1733258087225Flushing f3fb69afe3beda4eed21cce50a04d55d/c: creating writer at 1733258087236 (+11 ms)Flushing f3fb69afe3beda4eed21cce50a04d55d/c: appending metadata at 1733258087249 (+13 ms)Flushing f3fb69afe3beda4eed21cce50a04d55d/c: closing flushed file at 1733258087249Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52eda13f: reopening flushed file at 1733258087259 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1216d523: reopening flushed file at 1733258087265 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for f3fb69afe3beda4eed21cce50a04d55d in 62ms, sequenceid=35, compaction requested=false; wal=null at 1733258087272 (+7 ms)Cleaning up temporary data from old regions at 1733258087274 (+2 ms)Region opened successfully at 1733258087280 (+6 ms) 2024-12-03T20:34:47,337 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087176, size=0 (0bytes) 2024-12-03T20:34:47,338 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087176 might be still open, length is 0 2024-12-03T20:34:47,338 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087176 2024-12-03T20:34:47,338 WARN [IPC Server handler 2 on default port 39361 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087176 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-03T20:34:47,338 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087176 after 0ms 2024-12-03T20:34:47,353 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T20:34:48,564 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:46746 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:36623:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46746 dst: /127.0.0.1:36623 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36623 remote=/127.0.0.1:46746]. Total timeout mills is 60000, 58742 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:48,565 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:47134 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:46119:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47134 dst: /127.0.0.1:46119 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:48,565 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1711650157_22 at /127.0.0.1:35042 [Receiving block BP-1199604065-172.17.0.3-1733258049083:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:33321:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35042 dst: /127.0.0.1:33321 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:48,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741930_1112 (size=2936) 2024-12-03T20:34:48,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741930_1112 (size=2936) 2024-12-03T20:34:51,339 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087176 after 4001ms 2024-12-03T20:34:51,342 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087176: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:51,342 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087176 took 4005ms 2024-12-03T20:34:51,345 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087176; continuing. 2024-12-03T20:34:51,345 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087176 so closing down 2024-12-03T20:34:51,345 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T20:34:51,347 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733258087176.temp 2024-12-03T20:34:51,348 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000037-wal.1733258087176.temp 2024-12-03T20:34:51,349 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T20:34:51,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741933_1113 (size=2944) 2024-12-03T20:34:51,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741933_1113 (size=2944) 2024-12-03T20:34:51,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741933_1113 (size=2944) 2024-12-03T20:34:51,355 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000037-wal.1733258087176.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T20:34:51,356 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000037-wal.1733258087176.temp to hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000066 2024-12-03T20:34:51,356 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087176, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T20:34:51,357 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087176, journal: Splitting hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087176, size=0 (0bytes) at 1733258087338Finishing writing output for hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087176 so closing down at 1733258091345 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000037-wal.1733258087176.temp at 1733258091348 (+3 ms)3 split writer threads finished at 1733258091349 (+1 ms)Closed recovered edits writer path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000037-wal.1733258087176.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733258091355 (+6 ms)Rename recovered edits hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000037-wal.1733258087176.temp to hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000066 at 1733258091356 (+1 ms)Processed 30 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087176, size=0, length=0, corrupted=false, cancelled=false at 1733258091356 2024-12-03T20:34:51,358 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087176 to hdfs://localhost:39361/hbase/oldWALs/wal.1733258087176 2024-12-03T20:34:51,359 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000066 2024-12-03T20:34:51,359 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T20:34:51,360 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39361/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010, archiveDir=hdfs://localhost:39361/hbase/oldWALs, maxLogs=32 2024-12-03T20:34:51,371 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258091361, exclude list is [], retry=0 2024-12-03T20:34:51,373 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36623,DS-92eaa216-f151-4266-97d7-3e835c8da57f,DISK] 2024-12-03T20:34:51,373 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46119,DS-f520253d-8fa4-4dfa-9b7e-4820b895a7cd,DISK] 2024-12-03T20:34:51,374 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33321,DS-3e4dbf34-b9a5-4866-8916-93b51dbe96ae,DISK] 2024-12-03T20:34:51,375 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258091361 2024-12-03T20:34:51,375 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33459:33459),(127.0.0.1/127.0.0.1:34005:34005),(127.0.0.1/127.0.0.1:36683:36683)] 2024-12-03T20:34:51,375 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T20:34:51,378 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:51,379 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3fb69afe3beda4eed21cce50a04d55d columnFamilyName a 2024-12-03T20:34:51,379 DEBUG [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:51,384 DEBUG [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/a/97007ba3fbda4db5af9ab5f04dc70371 2024-12-03T20:34:51,384 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(327): Store=f3fb69afe3beda4eed21cce50a04d55d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:51,384 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:51,385 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3fb69afe3beda4eed21cce50a04d55d columnFamilyName b 2024-12-03T20:34:51,385 DEBUG [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:51,391 DEBUG [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/b/096c197683d54b6588ecf185b7f01a14 2024-12-03T20:34:51,391 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(327): Store=f3fb69afe3beda4eed21cce50a04d55d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:51,391 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:51,392 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3fb69afe3beda4eed21cce50a04d55d columnFamilyName c 2024-12-03T20:34:51,392 DEBUG [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T20:34:51,397 DEBUG [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/c/4ca8f46b79bf43f5820f4849feb1a3a6 2024-12-03T20:34:51,397 INFO [StoreOpener-f3fb69afe3beda4eed21cce50a04d55d-1 {}] regionserver.HStore(327): Store=f3fb69afe3beda4eed21cce50a04d55d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T20:34:51,397 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:51,398 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:51,399 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:51,400 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000066 2024-12-03T20:34:51,402 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T20:34:51,405 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000066 2024-12-03T20:34:51,405 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f3fb69afe3beda4eed21cce50a04d55d 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-03T20:34:51,417 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/.tmp/a/91ce931063da408991b86f4c99c7120b is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733258087287/Put/seqid=0 2024-12-03T20:34:51,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741935_1115 (size=5958) 2024-12-03T20:34:51,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741935_1115 (size=5958) 2024-12-03T20:34:51,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741935_1115 (size=5958) 2024-12-03T20:34:51,423 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/.tmp/a/91ce931063da408991b86f4c99c7120b 2024-12-03T20:34:51,445 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/.tmp/b/0ba523a954ba493280e875586c959a15 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733258087294/Put/seqid=0 2024-12-03T20:34:51,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741936_1116 (size=5958) 2024-12-03T20:34:51,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741936_1116 (size=5958) 2024-12-03T20:34:51,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741936_1116 (size=5958) 2024-12-03T20:34:51,451 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/.tmp/b/0ba523a954ba493280e875586c959a15 2024-12-03T20:34:51,467 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/.tmp/c/8be76b73b25b4b339fbaf8ebbf3b8610 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733258087300/Put/seqid=0 2024-12-03T20:34:51,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741937_1117 (size=5958) 2024-12-03T20:34:51,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741937_1117 (size=5958) 2024-12-03T20:34:51,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741937_1117 (size=5958) 2024-12-03T20:34:51,473 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/.tmp/c/8be76b73b25b4b339fbaf8ebbf3b8610 2024-12-03T20:34:51,477 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/.tmp/a/91ce931063da408991b86f4c99c7120b as hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/a/91ce931063da408991b86f4c99c7120b 2024-12-03T20:34:51,482 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/a/91ce931063da408991b86f4c99c7120b, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T20:34:51,483 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/.tmp/b/0ba523a954ba493280e875586c959a15 as hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/b/0ba523a954ba493280e875586c959a15 2024-12-03T20:34:51,487 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/b/0ba523a954ba493280e875586c959a15, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T20:34:51,488 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/.tmp/c/8be76b73b25b4b339fbaf8ebbf3b8610 as hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/c/8be76b73b25b4b339fbaf8ebbf3b8610 2024-12-03T20:34:51,492 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/c/8be76b73b25b4b339fbaf8ebbf3b8610, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T20:34:51,492 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for f3fb69afe3beda4eed21cce50a04d55d in 87ms, sequenceid=66, compaction requested=false; wal=null 2024-12-03T20:34:51,493 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/0000000000000000066 2024-12-03T20:34:51,494 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:51,494 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:51,494 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T20:34:51,495 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f3fb69afe3beda4eed21cce50a04d55d 2024-12-03T20:34:51,497 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/hbase/data/default/testReplayEditsWrittenViaHRegion/f3fb69afe3beda4eed21cce50a04d55d/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-03T20:34:51,498 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f3fb69afe3beda4eed21cce50a04d55d; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61843751, jitterRate=-0.07845629751682281}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T20:34:51,498 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f3fb69afe3beda4eed21cce50a04d55d: Writing region info on filesystem at 1733258091375Initializing all the Stores at 1733258091376 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258091376Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258091378 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733258091378Obtaining lock to block concurrent updates at 1733258091405 (+27 ms)Preparing flush snapshotting stores in f3fb69afe3beda4eed21cce50a04d55d at 1733258091405Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733258091405Flushing stores of testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d. at 1733258091405Flushing f3fb69afe3beda4eed21cce50a04d55d/a: creating writer at 1733258091405Flushing f3fb69afe3beda4eed21cce50a04d55d/a: appending metadata at 1733258091416 (+11 ms)Flushing f3fb69afe3beda4eed21cce50a04d55d/a: closing flushed file at 1733258091416Flushing f3fb69afe3beda4eed21cce50a04d55d/b: creating writer at 1733258091426 (+10 ms)Flushing f3fb69afe3beda4eed21cce50a04d55d/b: appending metadata at 1733258091444 (+18 ms)Flushing f3fb69afe3beda4eed21cce50a04d55d/b: closing flushed file at 1733258091444Flushing f3fb69afe3beda4eed21cce50a04d55d/c: creating writer at 1733258091455 (+11 ms)Flushing f3fb69afe3beda4eed21cce50a04d55d/c: appending metadata at 1733258091467 (+12 ms)Flushing f3fb69afe3beda4eed21cce50a04d55d/c: closing flushed file at 1733258091467Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c559297: reopening flushed file at 1733258091477 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3971cd89: reopening flushed file at 1733258091482 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c747bc7: reopening flushed file at 1733258091487 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for f3fb69afe3beda4eed21cce50a04d55d in 87ms, sequenceid=66, compaction requested=false; wal=null at 1733258091492 (+5 ms)Cleaning up temporary data from old regions at 1733258091494 (+2 ms)Region opened successfully at 1733258091498 (+4 ms) 2024-12-03T20:34:51,511 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f3fb69afe3beda4eed21cce50a04d55d, disabling compactions & flushes 2024-12-03T20:34:51,511 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d. 2024-12-03T20:34:51,511 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d. 2024-12-03T20:34:51,511 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d. after waiting 0 ms 2024-12-03T20:34:51,511 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d. 2024-12-03T20:34:51,512 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733258087011.f3fb69afe3beda4eed21cce50a04d55d. 2024-12-03T20:34:51,512 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f3fb69afe3beda4eed21cce50a04d55d: Waiting for close lock at 1733258091510Disabling compacts and flushes for region at 1733258091510Disabling writes for close at 1733258091511 (+1 ms)Writing region close event to WAL at 1733258091512 (+1 ms)Closed at 1733258091512 2024-12-03T20:34:51,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741934_1114 (size=95) 2024-12-03T20:34:51,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741934_1114 (size=95) 2024-12-03T20:34:51,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741934_1114 (size=95) 2024-12-03T20:34:51,516 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T20:34:51,516 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733258091361) 2024-12-03T20:34:51,529 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=449 (was 444) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1315935248_22 at /127.0.0.1:35070 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1315935248_22 at /127.0.0.1:47174 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1315935248_22 at /127.0.0.1:46782 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:39361 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:39361 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1408 (was 1342) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=202 (was 203), ProcessCount=11 (was 11), AvailableMemoryMB=1931 (was 1967) 2024-12-03T20:34:51,530 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1408 is superior to 1024 2024-12-03T20:34:51,530 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T20:34:51,530 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T20:34:51,530 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T20:34:51,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T20:34:51,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T20:34:51,531 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T20:34:51,531 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T20:34:51,531 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1122168874, stopped=false 2024-12-03T20:34:51,531 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b29c245002d9,38205,1733258053478 2024-12-03T20:34:51,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T20:34:51,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1019ce700170002, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T20:34:51,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35249-0x1019ce700170001, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T20:34:51,605 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T20:34:51,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:51,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1019ce700170002, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:51,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35249-0x1019ce700170001, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:51,606 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T20:34:51,606 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T20:34:51,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T20:34:51,606 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b29c245002d9,35249,1733258054237' ***** 2024-12-03T20:34:51,606 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T20:34:51,606 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b29c245002d9,44079,1733258054330' ***** 2024-12-03T20:34:51,606 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T20:34:51,606 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44079-0x1019ce700170002, quorum=127.0.0.1:49830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T20:34:51,606 INFO [RS:0;b29c245002d9:35249 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T20:34:51,607 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T20:34:51,607 INFO [RS:1;b29c245002d9:44079 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T20:34:51,607 INFO [RS:0;b29c245002d9:35249 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T20:34:51,607 INFO [RS:1;b29c245002d9:44079 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T20:34:51,607 INFO [RS:1;b29c245002d9:44079 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T20:34:51,607 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T20:34:51,607 INFO [RS:1;b29c245002d9:44079 {}] regionserver.HRegionServer(959): stopping server b29c245002d9,44079,1733258054330 2024-12-03T20:34:51,607 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T20:34:51,607 INFO [RS:1;b29c245002d9:44079 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T20:34:51,607 INFO [RS:1;b29c245002d9:44079 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;b29c245002d9:44079. 2024-12-03T20:34:51,607 INFO [RS:0;b29c245002d9:35249 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T20:34:51,607 DEBUG [RS:1;b29c245002d9:44079 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T20:34:51,607 DEBUG [RS:1;b29c245002d9:44079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T20:34:51,607 INFO [RS:0;b29c245002d9:35249 {}] regionserver.HRegionServer(3091): Received CLOSE for 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:51,607 INFO [RS:0;b29c245002d9:35249 {}] regionserver.HRegionServer(959): stopping server b29c245002d9,35249,1733258054237 2024-12-03T20:34:51,607 INFO [RS:1;b29c245002d9:44079 {}] regionserver.HRegionServer(976): stopping server b29c245002d9,44079,1733258054330; all regions closed. 2024-12-03T20:34:51,607 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35249-0x1019ce700170001, quorum=127.0.0.1:49830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T20:34:51,607 INFO [RS:0;b29c245002d9:35249 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T20:34:51,607 INFO [RS:0;b29c245002d9:35249 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b29c245002d9:35249. 2024-12-03T20:34:51,607 DEBUG [RS:0;b29c245002d9:35249 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T20:34:51,607 DEBUG [RS:0;b29c245002d9:35249 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T20:34:51,607 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2fcbcf91e026c1e68ac208aa693d488a, disabling compactions & flushes 2024-12-03T20:34:51,608 INFO [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:51,608 INFO [RS:0;b29c245002d9:35249 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T20:34:51,608 INFO [RS:0;b29c245002d9:35249 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T20:34:51,608 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:51,608 INFO [RS:0;b29c245002d9:35249 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T20:34:51,608 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. after waiting 0 ms 2024-12-03T20:34:51,608 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:51,608 INFO [RS:0;b29c245002d9:35249 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T20:34:51,608 INFO [RS:0;b29c245002d9:35249 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-03T20:34:51,608 DEBUG [RS:0;b29c245002d9:35249 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 2fcbcf91e026c1e68ac208aa693d488a=testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a.} 2024-12-03T20:34:51,608 DEBUG [RS:0;b29c245002d9:35249 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2fcbcf91e026c1e68ac208aa693d488a 2024-12-03T20:34:51,608 DEBUG [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T20:34:51,608 INFO [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T20:34:51,608 DEBUG [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T20:34:51,608 DEBUG [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T20:34:51,609 DEBUG [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T20:34:51,609 INFO [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-03T20:34:51,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741834_1010 (size=95) 2024-12-03T20:34:51,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741834_1010 (size=95) 2024-12-03T20:34:51,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741834_1010 (size=95) 2024-12-03T20:34:51,613 DEBUG [RS:1;b29c245002d9:44079 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/oldWALs 2024-12-03T20:34:51,613 INFO [RS:1;b29c245002d9:44079 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b29c245002d9%2C44079%2C1733258054330:(num 1733258055920) 2024-12-03T20:34:51,613 DEBUG [RS:1;b29c245002d9:44079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T20:34:51,613 INFO [RS:1;b29c245002d9:44079 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T20:34:51,613 INFO [RS:1;b29c245002d9:44079 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T20:34:51,613 INFO [RS:1;b29c245002d9:44079 {}] hbase.ChoreService(370): Chore service for: regionserver/b29c245002d9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T20:34:51,614 INFO [RS:1;b29c245002d9:44079 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T20:34:51,614 INFO [RS:1;b29c245002d9:44079 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T20:34:51,614 INFO [RS:1;b29c245002d9:44079 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T20:34:51,614 INFO [RS:1;b29c245002d9:44079 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T20:34:51,614 INFO [RS:1;b29c245002d9:44079 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44079 2024-12-03T20:34:51,614 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/default/testReplayEditsAfterRegionMovedWithMultiCF/2fcbcf91e026c1e68ac208aa693d488a/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-03T20:34:51,615 INFO [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:51,615 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2fcbcf91e026c1e68ac208aa693d488a: Waiting for close lock at 1733258091607Running coprocessor pre-close hooks at 1733258091607Disabling compacts and flushes for region at 1733258091607Disabling writes for close at 1733258091608 (+1 ms)Writing region close event to WAL at 1733258091609 (+1 ms)Running coprocessor post-close hooks at 1733258091615 (+6 ms)Closed at 1733258091615 2024-12-03T20:34:51,615 INFO [regionserver/b29c245002d9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T20:34:51,615 DEBUG [RS_CLOSE_REGION-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a. 2024-12-03T20:34:51,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1019ce700170002, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b29c245002d9,44079,1733258054330 2024-12-03T20:34:51,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T20:34:51,626 INFO [RS:1;b29c245002d9:44079 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T20:34:51,626 INFO [regionserver/b29c245002d9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T20:34:51,627 INFO [regionserver/b29c245002d9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T20:34:51,634 DEBUG [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740/.tmp/info/35710fa4e53c4f05babea53b31040549 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733258068729.2fcbcf91e026c1e68ac208aa693d488a./info:regioninfo/1733258071908/Put/seqid=0 2024-12-03T20:34:51,637 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b29c245002d9,44079,1733258054330] 2024-12-03T20:34:51,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741938_1118 (size=8243) 2024-12-03T20:34:51,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741938_1118 (size=8243) 2024-12-03T20:34:51,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741938_1118 (size=8243) 2024-12-03T20:34:51,640 INFO [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740/.tmp/info/35710fa4e53c4f05babea53b31040549 2024-12-03T20:34:51,647 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b29c245002d9,44079,1733258054330 already deleted, retry=false 2024-12-03T20:34:51,648 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b29c245002d9,44079,1733258054330 expired; onlineServers=1 2024-12-03T20:34:51,658 DEBUG [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740/.tmp/ns/334ddfd5d64a48bca4cf2d91c119a778 is 43, key is default/ns:d/1733258056573/Put/seqid=0 2024-12-03T20:34:51,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741939_1119 (size=5153) 2024-12-03T20:34:51,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741939_1119 (size=5153) 2024-12-03T20:34:51,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741939_1119 (size=5153) 2024-12-03T20:34:51,664 INFO [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740/.tmp/ns/334ddfd5d64a48bca4cf2d91c119a778 2024-12-03T20:34:51,683 DEBUG [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740/.tmp/table/9b644caa73cd454b8aabd9e5917685d0 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733258069150/Put/seqid=0 2024-12-03T20:34:51,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741940_1120 (size=5431) 2024-12-03T20:34:51,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741940_1120 (size=5431) 2024-12-03T20:34:51,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741940_1120 (size=5431) 2024-12-03T20:34:51,689 INFO [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740/.tmp/table/9b644caa73cd454b8aabd9e5917685d0 2024-12-03T20:34:51,694 DEBUG [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740/.tmp/info/35710fa4e53c4f05babea53b31040549 as hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740/info/35710fa4e53c4f05babea53b31040549 2024-12-03T20:34:51,699 INFO [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740/info/35710fa4e53c4f05babea53b31040549, entries=18, sequenceid=21, filesize=8.0 K 2024-12-03T20:34:51,700 DEBUG [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740/.tmp/ns/334ddfd5d64a48bca4cf2d91c119a778 as hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740/ns/334ddfd5d64a48bca4cf2d91c119a778 2024-12-03T20:34:51,705 INFO [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740/ns/334ddfd5d64a48bca4cf2d91c119a778, entries=2, sequenceid=21, filesize=5.0 K 2024-12-03T20:34:51,706 DEBUG [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740/.tmp/table/9b644caa73cd454b8aabd9e5917685d0 as hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740/table/9b644caa73cd454b8aabd9e5917685d0 2024-12-03T20:34:51,710 INFO [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740/table/9b644caa73cd454b8aabd9e5917685d0, entries=2, sequenceid=21, filesize=5.3 K 2024-12-03T20:34:51,711 INFO [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 102ms, sequenceid=21, compaction requested=false 2024-12-03T20:34:51,716 DEBUG [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-03T20:34:51,717 DEBUG [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T20:34:51,717 INFO [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T20:34:51,717 DEBUG [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733258091608Running coprocessor pre-close hooks at 1733258091608Disabling compacts and flushes for region at 1733258091608Disabling writes for close at 1733258091609 (+1 ms)Obtaining lock to block concurrent updates at 1733258091609Preparing flush snapshotting stores in 1588230740 at 1733258091609Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733258091609Flushing stores of hbase:meta,,1.1588230740 at 1733258091611 (+2 ms)Flushing 1588230740/info: creating writer at 1733258091611Flushing 1588230740/info: appending metadata at 1733258091633 (+22 ms)Flushing 1588230740/info: closing flushed file at 1733258091633Flushing 1588230740/ns: creating writer at 1733258091645 (+12 ms)Flushing 1588230740/ns: appending metadata at 1733258091657 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1733258091657Flushing 1588230740/table: creating writer at 1733258091669 (+12 ms)Flushing 1588230740/table: appending metadata at 1733258091683 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733258091683Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@679727fc: reopening flushed file at 1733258091694 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5777420c: reopening flushed file at 1733258091699 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@af454da: reopening flushed file at 1733258091705 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 102ms, sequenceid=21, compaction requested=false at 1733258091712 (+7 ms)Writing region close event to WAL at 1733258091713 (+1 ms)Running coprocessor post-close hooks at 1733258091717 (+4 ms)Closed at 1733258091717 2024-12-03T20:34:51,717 DEBUG [RS_CLOSE_META-regionserver/b29c245002d9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T20:34:51,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1019ce700170002, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T20:34:51,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1019ce700170002, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T20:34:51,737 INFO [RS:1;b29c245002d9:44079 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T20:34:51,737 INFO [RS:1;b29c245002d9:44079 {}] regionserver.HRegionServer(1031): Exiting; stopping=b29c245002d9,44079,1733258054330; zookeeper connection closed. 2024-12-03T20:34:51,738 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@53068462 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@53068462 2024-12-03T20:34:51,808 INFO [RS:0;b29c245002d9:35249 {}] regionserver.HRegionServer(976): stopping server b29c245002d9,35249,1733258054237; all regions closed. 2024-12-03T20:34:51,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741836_1012 (size=4706) 2024-12-03T20:34:51,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741836_1012 (size=4706) 2024-12-03T20:34:51,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741836_1012 (size=4706) 2024-12-03T20:34:51,813 DEBUG [RS:0;b29c245002d9:35249 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/oldWALs 2024-12-03T20:34:51,813 INFO [RS:0;b29c245002d9:35249 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b29c245002d9%2C35249%2C1733258054237.meta:.meta(num 1733258056426) 2024-12-03T20:34:51,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741833_1009 (size=2178) 2024-12-03T20:34:51,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741833_1009 (size=2178) 2024-12-03T20:34:51,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741833_1009 (size=2178) 2024-12-03T20:34:51,817 DEBUG [RS:0;b29c245002d9:35249 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/oldWALs 2024-12-03T20:34:51,817 INFO [RS:0;b29c245002d9:35249 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b29c245002d9%2C35249%2C1733258054237:(num 1733258055920) 2024-12-03T20:34:51,817 DEBUG [RS:0;b29c245002d9:35249 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T20:34:51,817 INFO [RS:0;b29c245002d9:35249 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T20:34:51,817 INFO [RS:0;b29c245002d9:35249 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T20:34:51,817 INFO [RS:0;b29c245002d9:35249 {}] hbase.ChoreService(370): Chore service for: regionserver/b29c245002d9:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T20:34:51,817 INFO [RS:0;b29c245002d9:35249 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T20:34:51,817 INFO [regionserver/b29c245002d9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T20:34:51,817 INFO [RS:0;b29c245002d9:35249 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35249 2024-12-03T20:34:51,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T20:34:51,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35249-0x1019ce700170001, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b29c245002d9,35249,1733258054237 2024-12-03T20:34:51,848 INFO [RS:0;b29c245002d9:35249 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T20:34:51,858 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b29c245002d9,35249,1733258054237] 2024-12-03T20:34:51,869 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b29c245002d9,35249,1733258054237 already deleted, retry=false 2024-12-03T20:34:51,869 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b29c245002d9,35249,1733258054237 expired; onlineServers=0 2024-12-03T20:34:51,869 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b29c245002d9,38205,1733258053478' ***** 2024-12-03T20:34:51,869 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T20:34:51,870 INFO [M:0;b29c245002d9:38205 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T20:34:51,870 INFO [M:0;b29c245002d9:38205 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T20:34:51,870 DEBUG [M:0;b29c245002d9:38205 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T20:34:51,871 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T20:34:51,871 DEBUG [M:0;b29c245002d9:38205 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T20:34:51,871 DEBUG [master/b29c245002d9:0:becomeActiveMaster-HFileCleaner.large.0-1733258055613 {}] cleaner.HFileCleaner(306): Exit Thread[master/b29c245002d9:0:becomeActiveMaster-HFileCleaner.large.0-1733258055613,5,FailOnTimeoutGroup] 2024-12-03T20:34:51,871 DEBUG [master/b29c245002d9:0:becomeActiveMaster-HFileCleaner.small.0-1733258055614 {}] cleaner.HFileCleaner(306): Exit Thread[master/b29c245002d9:0:becomeActiveMaster-HFileCleaner.small.0-1733258055614,5,FailOnTimeoutGroup] 2024-12-03T20:34:51,871 INFO [M:0;b29c245002d9:38205 {}] hbase.ChoreService(370): Chore service for: master/b29c245002d9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T20:34:51,872 INFO [M:0;b29c245002d9:38205 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T20:34:51,872 DEBUG [M:0;b29c245002d9:38205 {}] master.HMaster(1795): Stopping service threads 2024-12-03T20:34:51,872 INFO [M:0;b29c245002d9:38205 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T20:34:51,872 INFO [M:0;b29c245002d9:38205 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T20:34:51,874 INFO [M:0;b29c245002d9:38205 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T20:34:51,874 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T20:34:51,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T20:34:51,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T20:34:51,884 DEBUG [M:0;b29c245002d9:38205 {}] zookeeper.ZKUtil(347): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T20:34:51,884 WARN [M:0;b29c245002d9:38205 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T20:34:51,886 INFO [M:0;b29c245002d9:38205 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/.lastflushedseqids 2024-12-03T20:34:51,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741941_1121 (size=138) 2024-12-03T20:34:51,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741941_1121 (size=138) 2024-12-03T20:34:51,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741941_1121 (size=138) 2024-12-03T20:34:51,906 INFO [M:0;b29c245002d9:38205 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T20:34:51,906 INFO [M:0;b29c245002d9:38205 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T20:34:51,906 DEBUG [M:0;b29c245002d9:38205 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T20:34:51,906 INFO [M:0;b29c245002d9:38205 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T20:34:51,906 DEBUG [M:0;b29c245002d9:38205 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T20:34:51,906 DEBUG [M:0;b29c245002d9:38205 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T20:34:51,906 DEBUG [M:0;b29c245002d9:38205 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T20:34:51,906 INFO [M:0;b29c245002d9:38205 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-12-03T20:34:51,920 DEBUG [M:0;b29c245002d9:38205 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/82cb96c961cc4aef86d527689de065ac is 82, key is hbase:meta,,1/info:regioninfo/1733258056500/Put/seqid=0 2024-12-03T20:34:51,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741942_1122 (size=5672) 2024-12-03T20:34:51,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741942_1122 (size=5672) 2024-12-03T20:34:51,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741942_1122 (size=5672) 2024-12-03T20:34:51,925 INFO [M:0;b29c245002d9:38205 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/82cb96c961cc4aef86d527689de065ac 2024-12-03T20:34:51,945 DEBUG [M:0;b29c245002d9:38205 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d1de61b87d35425697df23e008e18f54 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733258069157/Put/seqid=0 2024-12-03T20:34:51,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741943_1123 (size=7755) 2024-12-03T20:34:51,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741943_1123 (size=7755) 2024-12-03T20:34:51,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741943_1123 (size=7755) 2024-12-03T20:34:51,952 INFO [M:0;b29c245002d9:38205 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d1de61b87d35425697df23e008e18f54 2024-12-03T20:34:51,956 INFO [M:0;b29c245002d9:38205 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d1de61b87d35425697df23e008e18f54 2024-12-03T20:34:51,958 INFO [RS:0;b29c245002d9:35249 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T20:34:51,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35249-0x1019ce700170001, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T20:34:51,958 INFO [RS:0;b29c245002d9:35249 {}] regionserver.HRegionServer(1031): Exiting; stopping=b29c245002d9,35249,1733258054237; zookeeper connection closed. 2024-12-03T20:34:51,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35249-0x1019ce700170001, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T20:34:51,959 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@122864de {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@122864de 2024-12-03T20:34:51,959 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-03T20:34:51,968 DEBUG [M:0;b29c245002d9:38205 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e05bc27960174e2497ba92b10a4732ab is 69, key is b29c245002d9,35249,1733258054237/rs:state/1733258055651/Put/seqid=0 2024-12-03T20:34:51,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741944_1124 (size=5440) 2024-12-03T20:34:51,975 INFO [M:0;b29c245002d9:38205 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e05bc27960174e2497ba92b10a4732ab 2024-12-03T20:34:51,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741944_1124 (size=5440) 2024-12-03T20:34:51,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741944_1124 (size=5440) 2024-12-03T20:34:51,979 INFO [M:0;b29c245002d9:38205 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e05bc27960174e2497ba92b10a4732ab 2024-12-03T20:34:51,980 DEBUG [M:0;b29c245002d9:38205 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/82cb96c961cc4aef86d527689de065ac as hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/82cb96c961cc4aef86d527689de065ac 2024-12-03T20:34:51,985 INFO [M:0;b29c245002d9:38205 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/82cb96c961cc4aef86d527689de065ac, entries=8, sequenceid=168, filesize=5.5 K 2024-12-03T20:34:51,986 DEBUG [M:0;b29c245002d9:38205 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d1de61b87d35425697df23e008e18f54 as hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d1de61b87d35425697df23e008e18f54 2024-12-03T20:34:51,991 INFO [M:0;b29c245002d9:38205 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d1de61b87d35425697df23e008e18f54 2024-12-03T20:34:51,991 INFO [M:0;b29c245002d9:38205 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d1de61b87d35425697df23e008e18f54, entries=17, sequenceid=168, filesize=7.6 K 2024-12-03T20:34:51,992 DEBUG [M:0;b29c245002d9:38205 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e05bc27960174e2497ba92b10a4732ab as hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e05bc27960174e2497ba92b10a4732ab 2024-12-03T20:34:51,996 INFO [M:0;b29c245002d9:38205 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e05bc27960174e2497ba92b10a4732ab 2024-12-03T20:34:51,996 INFO [M:0;b29c245002d9:38205 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39361/user/jenkins/test-data/76b777ab-2ec2-aa14-9d2e-09606cc6b596/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e05bc27960174e2497ba92b10a4732ab, entries=3, sequenceid=168, filesize=5.3 K 2024-12-03T20:34:51,997 INFO [M:0;b29c245002d9:38205 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 91ms, sequenceid=168, compaction requested=false 2024-12-03T20:34:51,999 INFO [M:0;b29c245002d9:38205 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T20:34:51,999 DEBUG [M:0;b29c245002d9:38205 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733258091906Disabling compacts and flushes for region at 1733258091906Disabling writes for close at 1733258091906Obtaining lock to block concurrent updates at 1733258091906Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733258091906Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1733258091906Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733258091907 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733258091907Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733258091919 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733258091919Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733258091930 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733258091944 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733258091944Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733258091956 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733258091968 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733258091968Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@103b4544: reopening flushed file at 1733258091979 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@641716f3: reopening flushed file at 1733258091986 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@161356e2: reopening flushed file at 1733258091991 (+5 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 91ms, sequenceid=168, compaction requested=false at 1733258091997 (+6 ms)Writing region close event to WAL at 1733258091999 (+2 ms)Closed at 1733258091999 2024-12-03T20:34:52,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46119 is added to blk_1073741830_1006 (size=56570) 2024-12-03T20:34:52,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33321 is added to blk_1073741830_1006 (size=56570) 2024-12-03T20:34:52,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36623 is added to blk_1073741830_1006 (size=56570) 2024-12-03T20:34:52,001 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T20:34:52,001 INFO [M:0;b29c245002d9:38205 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T20:34:52,001 INFO [M:0;b29c245002d9:38205 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38205 2024-12-03T20:34:52,002 INFO [M:0;b29c245002d9:38205 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T20:34:52,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T20:34:52,110 INFO [M:0;b29c245002d9:38205 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T20:34:52,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38205-0x1019ce700170000, quorum=127.0.0.1:49830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T20:34:52,120 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175/wal.1733258072459 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175/wal.1733258072459 (inode 16655) Holder DFSClient_NONMAPREDUCE_1711650157_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733258072175/wal.1733258072459 (inode 16655) Holder DFSClient_NONMAPREDUCE_1711650157_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T20:34:52,121 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733258064366/wal.1733258064464 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:52,122 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733258087010/wal.1733258087176 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:52,124 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589/wal.1733258081866 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589/wal.1733258081866 (inode 16678) Holder DFSClient_NONMAPREDUCE_1711650157_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733258072589/wal.1733258081866 (inode 16678) Holder DFSClient_NONMAPREDUCE_1711650157_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T20:34:52,125 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733258057543/wal.1733258057604 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:52,126 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041/wal.1733258082478 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041/wal.1733258082478 (inode 16704) Holder DFSClient_NONMAPREDUCE_1711650157_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733258082041/wal.1733258082478 (inode 16704) Holder DFSClient_NONMAPREDUCE_1711650157_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T20:34:52,127 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733258082609/wal.1733258082660 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T20:34:52,129 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal.1733258057419 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal.1733258057419 (inode 16485) Holder DFSClient_NONMAPREDUCE_1711650157_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733258057251/wal.1733258057419 (inode 16485) Holder DFSClient_NONMAPREDUCE_1711650157_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T20:34:52,131 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal.1733258057046 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal.1733258057046 (inode 16462) Holder DFSClient_NONMAPREDUCE_1711650157_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733258056820/wal.1733258057046 (inode 16462) Holder DFSClient_NONMAPREDUCE_1711650157_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T20:34:52,136 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@35f1150e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T20:34:52,137 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T20:34:52,137 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T20:34:52,137 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T20:34:52,138 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/hadoop.log.dir/,STOPPED} 2024-12-03T20:34:52,140 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T20:34:52,140 WARN [BP-1199604065-172.17.0.3-1733258049083 heartbeating to localhost/127.0.0.1:39361 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T20:34:52,140 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T20:34:52,140 WARN [BP-1199604065-172.17.0.3-1733258049083 heartbeating to localhost/127.0.0.1:39361 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1199604065-172.17.0.3-1733258049083 (Datanode Uuid d1035785-19d2-46d8-8b47-e021c23353d0) service to localhost/127.0.0.1:39361 2024-12-03T20:34:52,141 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/data/data5/current/BP-1199604065-172.17.0.3-1733258049083 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T20:34:52,141 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/data/data6/current/BP-1199604065-172.17.0.3-1733258049083 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T20:34:52,141 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T20:34:52,143 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7bd427b8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T20:34:52,143 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T20:34:52,143 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T20:34:52,144 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T20:34:52,144 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/hadoop.log.dir/,STOPPED} 2024-12-03T20:34:52,145 WARN [BP-1199604065-172.17.0.3-1733258049083 heartbeating to localhost/127.0.0.1:39361 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T20:34:52,145 WARN [BP-1199604065-172.17.0.3-1733258049083 heartbeating to localhost/127.0.0.1:39361 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1199604065-172.17.0.3-1733258049083 (Datanode Uuid 0eb744f4-f008-4db8-8d56-69feb009b4cb) service to localhost/127.0.0.1:39361 2024-12-03T20:34:52,145 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T20:34:52,145 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T20:34:52,145 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/data/data3/current/BP-1199604065-172.17.0.3-1733258049083 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T20:34:52,145 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/data/data4/current/BP-1199604065-172.17.0.3-1733258049083 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T20:34:52,145 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T20:34:52,147 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@330740de{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T20:34:52,148 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T20:34:52,148 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T20:34:52,148 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T20:34:52,148 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/hadoop.log.dir/,STOPPED} 2024-12-03T20:34:52,149 WARN [BP-1199604065-172.17.0.3-1733258049083 heartbeating to localhost/127.0.0.1:39361 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T20:34:52,149 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T20:34:52,149 WARN [BP-1199604065-172.17.0.3-1733258049083 heartbeating to localhost/127.0.0.1:39361 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1199604065-172.17.0.3-1733258049083 (Datanode Uuid c3badc97-6773-498e-871f-361f9428500c) service to localhost/127.0.0.1:39361 2024-12-03T20:34:52,149 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T20:34:52,150 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/data/data1/current/BP-1199604065-172.17.0.3-1733258049083 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T20:34:52,150 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/cluster_3435ee5d-e1ee-71f9-e505-98a3697d78d1/data/data2/current/BP-1199604065-172.17.0.3-1733258049083 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T20:34:52,150 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T20:34:52,156 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3717288f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T20:34:52,157 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T20:34:52,157 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T20:34:52,157 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T20:34:52,157 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d24b105a-ff28-9029-e635-e559e4c47fb7/hadoop.log.dir/,STOPPED} 2024-12-03T20:34:52,165 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T20:34:52,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down