2024-12-10 20:28:11,334 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-10 20:28:11,349 main DEBUG Took 0.013062 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-10 20:28:11,350 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-10 20:28:11,351 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-10 20:28:11,353 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-10 20:28:11,355 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:28:11,366 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-10 20:28:11,380 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:28:11,382 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:28:11,382 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:28:11,383 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:28:11,384 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:28:11,384 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:28:11,386 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:28:11,387 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:28:11,387 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:28:11,388 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:28:11,389 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:28:11,389 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:28:11,390 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:28:11,390 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:28:11,391 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:28:11,391 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:28:11,392 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:28:11,392 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:28:11,393 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:28:11,393 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:28:11,394 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:28:11,394 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:28:11,395 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:28:11,395 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 20:28:11,396 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:28:11,396 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-10 20:28:11,398 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 20:28:11,400 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-10 20:28:11,402 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-10 20:28:11,403 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-10 20:28:11,404 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-10 20:28:11,405 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-10 20:28:11,416 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-10 20:28:11,419 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-10 20:28:11,422 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-10 20:28:11,422 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-10 20:28:11,423 main DEBUG createAppenders(={Console}) 2024-12-10 20:28:11,424 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-10 20:28:11,424 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-10 20:28:11,424 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-10 20:28:11,425 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-10 20:28:11,425 main DEBUG OutputStream closed 2024-12-10 20:28:11,425 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-10 20:28:11,426 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-10 20:28:11,426 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-10 20:28:11,511 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-10 20:28:11,514 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-10 20:28:11,516 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-10 20:28:11,518 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-10 20:28:11,518 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-10 20:28:11,519 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-10 20:28:11,519 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-10 20:28:11,520 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-10 20:28:11,520 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-10 20:28:11,521 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-10 20:28:11,521 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-10 20:28:11,522 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-10 20:28:11,522 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-10 20:28:11,522 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-10 20:28:11,523 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-10 20:28:11,523 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-10 20:28:11,523 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-10 20:28:11,525 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-10 20:28:11,527 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10 20:28:11,528 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-10 20:28:11,528 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-10 20:28:11,529 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-10T20:28:11,819 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19 2024-12-10 20:28:11,823 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-10 20:28:11,823 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10T20:28:11,833 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-10T20:28:11,841 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayCompressed timeout: 13 mins 2024-12-10T20:28:11,864 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-10T20:28:11,913 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-10T20:28:11,913 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-10T20:28:11,924 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-10T20:28:11,936 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07, deleteOnExit=true 2024-12-10T20:28:11,936 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-10T20:28:11,937 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/test.cache.data in system properties and HBase conf 2024-12-10T20:28:11,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/hadoop.tmp.dir in system properties and HBase conf 2024-12-10T20:28:11,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/hadoop.log.dir in system properties and HBase conf 2024-12-10T20:28:11,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-10T20:28:11,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-10T20:28:11,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-10T20:28:12,004 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-10T20:28:12,086 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T20:28:12,090 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-10T20:28:12,090 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-10T20:28:12,091 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-10T20:28:12,091 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-10T20:28:12,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-10T20:28:12,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-10T20:28:12,093 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-10T20:28:12,093 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-10T20:28:12,094 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-10T20:28:12,094 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/nfs.dump.dir in system properties and HBase conf 2024-12-10T20:28:12,095 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/java.io.tmpdir in system properties and HBase conf 2024-12-10T20:28:12,095 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-10T20:28:12,096 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-10T20:28:12,096 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-10T20:28:13,437 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-10T20:28:13,535 INFO [Time-limited test {}] log.Log(170): Logging initialized @3061ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-10T20:28:13,598 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T20:28:13,659 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T20:28:13,685 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T20:28:13,685 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T20:28:13,687 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-10T20:28:13,701 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T20:28:13,705 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/hadoop.log.dir/,AVAILABLE} 2024-12-10T20:28:13,707 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T20:28:13,886 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3717288f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/java.io.tmpdir/jetty-localhost-43075-hadoop-hdfs-3_4_1-tests_jar-_-any-14093699806144805144/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-10T20:28:13,891 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:43075} 2024-12-10T20:28:13,892 INFO [Time-limited test {}] server.Server(415): Started @3417ms 2024-12-10T20:28:14,664 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T20:28:14,674 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T20:28:14,676 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T20:28:14,676 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T20:28:14,677 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-10T20:28:14,681 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/hadoop.log.dir/,AVAILABLE} 2024-12-10T20:28:14,682 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T20:28:14,781 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@330740de{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/java.io.tmpdir/jetty-localhost-35891-hadoop-hdfs-3_4_1-tests_jar-_-any-6094747817180110475/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T20:28:14,782 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:35891} 2024-12-10T20:28:14,783 INFO [Time-limited test {}] server.Server(415): Started @4308ms 2024-12-10T20:28:14,832 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-10T20:28:14,945 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T20:28:14,954 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T20:28:14,957 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T20:28:14,957 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T20:28:14,957 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-10T20:28:14,958 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/hadoop.log.dir/,AVAILABLE} 2024-12-10T20:28:14,959 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T20:28:15,091 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bd427b8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/java.io.tmpdir/jetty-localhost-33469-hadoop-hdfs-3_4_1-tests_jar-_-any-17135794014156540590/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T20:28:15,092 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:33469} 2024-12-10T20:28:15,092 INFO [Time-limited test {}] server.Server(415): Started @4618ms 2024-12-10T20:28:15,094 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-10T20:28:15,129 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-10T20:28:15,133 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-10T20:28:15,136 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-10T20:28:15,137 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-10T20:28:15,137 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-10T20:28:15,138 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/hadoop.log.dir/,AVAILABLE} 2024-12-10T20:28:15,138 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-10T20:28:15,233 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@35f1150e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/java.io.tmpdir/jetty-localhost-44555-hadoop-hdfs-3_4_1-tests_jar-_-any-9384838704176141720/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T20:28:15,234 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:44555} 2024-12-10T20:28:15,234 INFO [Time-limited test {}] server.Server(415): Started @4760ms 2024-12-10T20:28:15,236 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-10T20:28:16,671 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/data/data2/current/BP-76190982-172.17.0.3-1733862492788/current, will proceed with Du for space computation calculation, 2024-12-10T20:28:16,671 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/data/data1/current/BP-76190982-172.17.0.3-1733862492788/current, will proceed with Du for space computation calculation, 2024-12-10T20:28:16,671 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/data/data4/current/BP-76190982-172.17.0.3-1733862492788/current, will proceed with Du for space computation calculation, 2024-12-10T20:28:16,671 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/data/data3/current/BP-76190982-172.17.0.3-1733862492788/current, will proceed with Du for space computation calculation, 2024-12-10T20:28:16,702 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-10T20:28:16,703 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-10T20:28:16,743 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc54af0d208cfba73 with lease ID 0xed08c3a14f9573cf: Processing first storage report for DS-ab2a9f30-80ab-4099-9aba-2b96709a16db from datanode DatanodeRegistration(127.0.0.1:40165, datanodeUuid=06602c36-db2d-42b9-8c1f-9aba903bd8fd, infoPort=46745, infoSecurePort=0, ipcPort=43849, storageInfo=lv=-57;cid=testClusterID;nsid=191662973;c=1733862492788) 2024-12-10T20:28:16,744 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc54af0d208cfba73 with lease ID 0xed08c3a14f9573cf: from storage DS-ab2a9f30-80ab-4099-9aba-2b96709a16db node DatanodeRegistration(127.0.0.1:40165, datanodeUuid=06602c36-db2d-42b9-8c1f-9aba903bd8fd, infoPort=46745, infoSecurePort=0, ipcPort=43849, storageInfo=lv=-57;cid=testClusterID;nsid=191662973;c=1733862492788), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-10T20:28:16,745 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfa645d126f07b8e8 with lease ID 0xed08c3a14f9573d0: Processing first storage report for DS-64c1773a-d76f-453f-9ced-7a9998f1aa65 from datanode DatanodeRegistration(127.0.0.1:45577, datanodeUuid=16b431a4-85e7-433c-add7-0b3e71221b62, infoPort=40461, infoSecurePort=0, ipcPort=32823, storageInfo=lv=-57;cid=testClusterID;nsid=191662973;c=1733862492788) 2024-12-10T20:28:16,745 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfa645d126f07b8e8 with lease ID 0xed08c3a14f9573d0: from storage DS-64c1773a-d76f-453f-9ced-7a9998f1aa65 node DatanodeRegistration(127.0.0.1:45577, datanodeUuid=16b431a4-85e7-433c-add7-0b3e71221b62, infoPort=40461, infoSecurePort=0, ipcPort=32823, storageInfo=lv=-57;cid=testClusterID;nsid=191662973;c=1733862492788), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-10T20:28:16,745 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc54af0d208cfba73 with lease ID 0xed08c3a14f9573cf: Processing first storage report for DS-db5313cc-1587-4967-bb97-19cf8d7cda20 from datanode DatanodeRegistration(127.0.0.1:40165, datanodeUuid=06602c36-db2d-42b9-8c1f-9aba903bd8fd, infoPort=46745, infoSecurePort=0, ipcPort=43849, storageInfo=lv=-57;cid=testClusterID;nsid=191662973;c=1733862492788) 2024-12-10T20:28:16,745 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc54af0d208cfba73 with lease ID 0xed08c3a14f9573cf: from storage DS-db5313cc-1587-4967-bb97-19cf8d7cda20 node DatanodeRegistration(127.0.0.1:40165, datanodeUuid=06602c36-db2d-42b9-8c1f-9aba903bd8fd, infoPort=46745, infoSecurePort=0, ipcPort=43849, storageInfo=lv=-57;cid=testClusterID;nsid=191662973;c=1733862492788), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-10T20:28:16,745 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfa645d126f07b8e8 with lease ID 0xed08c3a14f9573d0: Processing first storage report for DS-effcd830-527c-41ae-9870-681f21a9751c from datanode DatanodeRegistration(127.0.0.1:45577, datanodeUuid=16b431a4-85e7-433c-add7-0b3e71221b62, infoPort=40461, infoSecurePort=0, ipcPort=32823, storageInfo=lv=-57;cid=testClusterID;nsid=191662973;c=1733862492788) 2024-12-10T20:28:16,745 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfa645d126f07b8e8 with lease ID 0xed08c3a14f9573d0: from storage DS-effcd830-527c-41ae-9870-681f21a9751c node DatanodeRegistration(127.0.0.1:45577, datanodeUuid=16b431a4-85e7-433c-add7-0b3e71221b62, infoPort=40461, infoSecurePort=0, ipcPort=32823, storageInfo=lv=-57;cid=testClusterID;nsid=191662973;c=1733862492788), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-10T20:28:16,845 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/data/data6/current/BP-76190982-172.17.0.3-1733862492788/current, will proceed with Du for space computation calculation, 2024-12-10T20:28:16,845 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/data/data5/current/BP-76190982-172.17.0.3-1733862492788/current, will proceed with Du for space computation calculation, 2024-12-10T20:28:16,860 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-10T20:28:16,864 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9e09c0461a005441 with lease ID 0xed08c3a14f9573d1: Processing first storage report for DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe from datanode DatanodeRegistration(127.0.0.1:36025, datanodeUuid=60303f5d-7098-4533-bb59-60644eb83420, infoPort=33825, infoSecurePort=0, ipcPort=37367, storageInfo=lv=-57;cid=testClusterID;nsid=191662973;c=1733862492788) 2024-12-10T20:28:16,865 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9e09c0461a005441 with lease ID 0xed08c3a14f9573d1: from storage DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe node DatanodeRegistration(127.0.0.1:36025, datanodeUuid=60303f5d-7098-4533-bb59-60644eb83420, infoPort=33825, infoSecurePort=0, ipcPort=37367, storageInfo=lv=-57;cid=testClusterID;nsid=191662973;c=1733862492788), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-10T20:28:16,865 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9e09c0461a005441 with lease ID 0xed08c3a14f9573d1: Processing first storage report for DS-1b60f97b-36f1-4e3a-8f46-cb5fc549df25 from datanode DatanodeRegistration(127.0.0.1:36025, datanodeUuid=60303f5d-7098-4533-bb59-60644eb83420, infoPort=33825, infoSecurePort=0, ipcPort=37367, storageInfo=lv=-57;cid=testClusterID;nsid=191662973;c=1733862492788) 2024-12-10T20:28:16,865 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9e09c0461a005441 with lease ID 0xed08c3a14f9573d1: from storage DS-1b60f97b-36f1-4e3a-8f46-cb5fc549df25 node DatanodeRegistration(127.0.0.1:36025, datanodeUuid=60303f5d-7098-4533-bb59-60644eb83420, infoPort=33825, infoSecurePort=0, ipcPort=37367, storageInfo=lv=-57;cid=testClusterID;nsid=191662973;c=1733862492788), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-10T20:28:16,964 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19 2024-12-10T20:28:17,035 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/zookeeper_0, clientPort=53600, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-10T20:28:17,042 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53600 2024-12-10T20:28:17,055 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T20:28:17,058 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T20:28:17,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741825_1001 (size=7) 2024-12-10T20:28:17,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741825_1001 (size=7) 2024-12-10T20:28:17,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741825_1001 (size=7) 2024-12-10T20:28:17,651 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b with version=8 2024-12-10T20:28:17,651 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/hbase-staging 2024-12-10T20:28:17,939 INFO [Time-limited test {}] client.ConnectionUtils(128): master/03a2ae042c95:0 server-side Connection retries=45 2024-12-10T20:28:17,947 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T20:28:17,948 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T20:28:17,953 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T20:28:17,953 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T20:28:17,953 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T20:28:18,074 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-10T20:28:18,127 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-10T20:28:18,135 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-10T20:28:18,138 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T20:28:18,162 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 59800 (auto-detected) 2024-12-10T20:28:18,163 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-10T20:28:18,180 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34471 2024-12-10T20:28:18,198 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34471 connecting to ZooKeeper ensemble=127.0.0.1:53600 2024-12-10T20:28:18,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:344710x0, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T20:28:18,322 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34471-0x100116a21c40000 connected 2024-12-10T20:28:18,421 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T20:28:18,428 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T20:28:18,440 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T20:28:18,444 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b, hbase.cluster.distributed=false 2024-12-10T20:28:18,466 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T20:28:18,471 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34471 2024-12-10T20:28:18,472 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34471 2024-12-10T20:28:18,473 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34471 2024-12-10T20:28:18,473 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34471 2024-12-10T20:28:18,474 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34471 2024-12-10T20:28:18,571 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/03a2ae042c95:0 server-side Connection retries=45 2024-12-10T20:28:18,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T20:28:18,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T20:28:18,573 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T20:28:18,573 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T20:28:18,573 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T20:28:18,575 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T20:28:18,578 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T20:28:18,579 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34629 2024-12-10T20:28:18,580 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34629 connecting to ZooKeeper ensemble=127.0.0.1:53600 2024-12-10T20:28:18,581 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T20:28:18,583 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T20:28:18,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:346290x0, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T20:28:18,598 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:346290x0, quorum=127.0.0.1:53600, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T20:28:18,598 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34629-0x100116a21c40001 connected 2024-12-10T20:28:18,603 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T20:28:18,611 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T20:28:18,613 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34629-0x100116a21c40001, quorum=127.0.0.1:53600, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T20:28:18,618 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34629-0x100116a21c40001, quorum=127.0.0.1:53600, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T20:28:18,618 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34629 2024-12-10T20:28:18,619 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34629 2024-12-10T20:28:18,619 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34629 2024-12-10T20:28:18,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34629 2024-12-10T20:28:18,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34629 2024-12-10T20:28:18,634 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/03a2ae042c95:0 server-side Connection retries=45 2024-12-10T20:28:18,635 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T20:28:18,635 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T20:28:18,635 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T20:28:18,635 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T20:28:18,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T20:28:18,636 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T20:28:18,636 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T20:28:18,637 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41599 2024-12-10T20:28:18,638 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41599 connecting to ZooKeeper ensemble=127.0.0.1:53600 2024-12-10T20:28:18,639 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T20:28:18,641 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T20:28:18,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:415990x0, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T20:28:18,651 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:415990x0, quorum=127.0.0.1:53600, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T20:28:18,651 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41599-0x100116a21c40002 connected 2024-12-10T20:28:18,651 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T20:28:18,653 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T20:28:18,654 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41599-0x100116a21c40002, quorum=127.0.0.1:53600, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T20:28:18,656 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41599-0x100116a21c40002, quorum=127.0.0.1:53600, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T20:28:18,656 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41599 2024-12-10T20:28:18,657 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41599 2024-12-10T20:28:18,657 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41599 2024-12-10T20:28:18,659 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41599 2024-12-10T20:28:18,659 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41599 2024-12-10T20:28:18,673 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/03a2ae042c95:0 server-side Connection retries=45 2024-12-10T20:28:18,673 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T20:28:18,674 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T20:28:18,674 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T20:28:18,674 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T20:28:18,674 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T20:28:18,675 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T20:28:18,675 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T20:28:18,677 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37285 2024-12-10T20:28:18,679 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37285 connecting to ZooKeeper ensemble=127.0.0.1:53600 2024-12-10T20:28:18,680 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T20:28:18,683 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T20:28:18,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:372850x0, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T20:28:18,701 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:372850x0, quorum=127.0.0.1:53600, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T20:28:18,701 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37285-0x100116a21c40003 connected 2024-12-10T20:28:18,702 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T20:28:18,702 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T20:28:18,703 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37285-0x100116a21c40003, quorum=127.0.0.1:53600, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T20:28:18,705 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37285-0x100116a21c40003, quorum=127.0.0.1:53600, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T20:28:18,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37285 2024-12-10T20:28:18,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37285 2024-12-10T20:28:18,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37285 2024-12-10T20:28:18,710 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37285 2024-12-10T20:28:18,710 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37285 2024-12-10T20:28:18,726 DEBUG [M:0;03a2ae042c95:34471 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;03a2ae042c95:34471 2024-12-10T20:28:18,727 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/03a2ae042c95,34471,1733862497789 2024-12-10T20:28:18,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41599-0x100116a21c40002, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T20:28:18,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37285-0x100116a21c40003, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T20:28:18,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T20:28:18,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34629-0x100116a21c40001, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T20:28:18,747 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/03a2ae042c95,34471,1733862497789 2024-12-10T20:28:18,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34629-0x100116a21c40001, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T20:28:18,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37285-0x100116a21c40003, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T20:28:18,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41599-0x100116a21c40002, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T20:28:18,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41599-0x100116a21c40002, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:18,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34629-0x100116a21c40001, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:18,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:18,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37285-0x100116a21c40003, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:18,854 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T20:28:18,855 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/03a2ae042c95,34471,1733862497789 from backup master directory 2024-12-10T20:28:18,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41599-0x100116a21c40002, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T20:28:18,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37285-0x100116a21c40003, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T20:28:18,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/03a2ae042c95,34471,1733862497789 2024-12-10T20:28:18,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T20:28:18,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34629-0x100116a21c40001, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T20:28:18,970 WARN [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T20:28:18,970 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=03a2ae042c95,34471,1733862497789 2024-12-10T20:28:18,974 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-10T20:28:18,976 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-10T20:28:19,027 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/hbase.id] with ID: 534d3ae7-c7ac-4fbf-af98-6ae9e35d04c4 2024-12-10T20:28:19,027 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/.tmp/hbase.id 2024-12-10T20:28:19,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741826_1002 (size=42) 2024-12-10T20:28:19,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741826_1002 (size=42) 2024-12-10T20:28:19,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741826_1002 (size=42) 2024-12-10T20:28:19,041 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/.tmp/hbase.id]:[hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/hbase.id] 2024-12-10T20:28:19,089 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-10T20:28:19,093 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-10T20:28:19,110 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-10T20:28:19,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:19,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37285-0x100116a21c40003, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:19,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34629-0x100116a21c40001, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:19,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41599-0x100116a21c40002, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:19,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741827_1003 (size=196) 2024-12-10T20:28:19,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741827_1003 (size=196) 2024-12-10T20:28:19,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741827_1003 (size=196) 2024-12-10T20:28:19,170 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T20:28:19,172 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-10T20:28:19,177 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:19,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741828_1004 (size=1189) 2024-12-10T20:28:19,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741828_1004 (size=1189) 2024-12-10T20:28:19,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741828_1004 (size=1189) 2024-12-10T20:28:19,220 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/data/master/store 2024-12-10T20:28:19,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741829_1005 (size=34) 2024-12-10T20:28:19,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741829_1005 (size=34) 2024-12-10T20:28:19,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741829_1005 (size=34) 2024-12-10T20:28:19,249 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-10T20:28:19,253 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:19,255 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T20:28:19,255 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T20:28:19,256 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T20:28:19,258 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T20:28:19,258 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T20:28:19,258 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T20:28:19,260 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733862499255Disabling compacts and flushes for region at 1733862499255Disabling writes for close at 1733862499258 (+3 ms)Writing region close event to WAL at 1733862499258Closed at 1733862499258 2024-12-10T20:28:19,263 WARN [master/03a2ae042c95:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/data/master/store/.initializing 2024-12-10T20:28:19,263 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/WALs/03a2ae042c95,34471,1733862497789 2024-12-10T20:28:19,273 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T20:28:19,292 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=03a2ae042c95%2C34471%2C1733862497789, suffix=, logDir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/WALs/03a2ae042c95,34471,1733862497789, archiveDir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/oldWALs, maxLogs=10 2024-12-10T20:28:19,320 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/WALs/03a2ae042c95,34471,1733862497789/03a2ae042c95%2C34471%2C1733862497789.1733862499297, exclude list is [], retry=0 2024-12-10T20:28:19,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:19,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:19,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:19,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:19,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-10T20:28:19,379 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/WALs/03a2ae042c95,34471,1733862497789/03a2ae042c95%2C34471%2C1733862497789.1733862499297 2024-12-10T20:28:19,380 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:40461:40461)] 2024-12-10T20:28:19,380 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:19,381 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:19,383 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:28:19,384 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:28:19,417 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:28:19,437 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-10T20:28:19,440 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:19,442 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T20:28:19,443 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:28:19,446 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-10T20:28:19,446 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:19,447 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:19,447 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:28:19,450 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-10T20:28:19,450 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:19,451 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:19,451 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:28:19,454 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-10T20:28:19,454 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:19,455 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:19,455 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:28:19,458 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:28:19,459 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:28:19,464 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:28:19,465 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:28:19,468 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T20:28:19,471 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T20:28:19,476 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:28:19,477 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60708901, jitterRate=-0.09536688029766083}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T20:28:19,484 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733862499395Initializing all the Stores at 1733862499397 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733862499397Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862499398 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862499398Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862499398Cleaning up temporary data from old regions at 1733862499465 (+67 ms)Region opened successfully at 1733862499484 (+19 ms) 2024-12-10T20:28:19,485 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-10T20:28:19,518 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a730373, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=03a2ae042c95/172.17.0.3:0 2024-12-10T20:28:19,553 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-10T20:28:19,563 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-10T20:28:19,564 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-10T20:28:19,566 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-10T20:28:19,567 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-10T20:28:19,572 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-10T20:28:19,573 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-10T20:28:19,601 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-10T20:28:19,609 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-10T20:28:19,660 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-10T20:28:19,662 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-10T20:28:19,664 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-10T20:28:19,670 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-10T20:28:19,672 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-10T20:28:19,676 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-10T20:28:19,681 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-10T20:28:19,682 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-10T20:28:19,691 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-10T20:28:19,710 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-10T20:28:19,723 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-10T20:28:19,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37285-0x100116a21c40003, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T20:28:19,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41599-0x100116a21c40002, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T20:28:19,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T20:28:19,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:19,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37285-0x100116a21c40003, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:19,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41599-0x100116a21c40002, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:19,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34629-0x100116a21c40001, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T20:28:19,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34629-0x100116a21c40001, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:19,738 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=03a2ae042c95,34471,1733862497789, sessionid=0x100116a21c40000, setting cluster-up flag (Was=false) 2024-12-10T20:28:19,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34629-0x100116a21c40001, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:19,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37285-0x100116a21c40003, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:19,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41599-0x100116a21c40002, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:19,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:19,797 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-10T20:28:19,800 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=03a2ae042c95,34471,1733862497789 2024-12-10T20:28:19,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34629-0x100116a21c40001, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:19,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41599-0x100116a21c40002, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:19,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37285-0x100116a21c40003, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:19,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:19,849 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-10T20:28:19,852 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=03a2ae042c95,34471,1733862497789 2024-12-10T20:28:19,864 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-10T20:28:19,914 INFO [RS:0;03a2ae042c95:34629 {}] regionserver.HRegionServer(746): ClusterId : 534d3ae7-c7ac-4fbf-af98-6ae9e35d04c4 2024-12-10T20:28:19,914 INFO [RS:1;03a2ae042c95:41599 {}] regionserver.HRegionServer(746): ClusterId : 534d3ae7-c7ac-4fbf-af98-6ae9e35d04c4 2024-12-10T20:28:19,914 INFO [RS:2;03a2ae042c95:37285 {}] regionserver.HRegionServer(746): ClusterId : 534d3ae7-c7ac-4fbf-af98-6ae9e35d04c4 2024-12-10T20:28:19,916 DEBUG [RS:1;03a2ae042c95:41599 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T20:28:19,916 DEBUG [RS:0;03a2ae042c95:34629 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T20:28:19,916 DEBUG [RS:2;03a2ae042c95:37285 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T20:28:19,925 DEBUG [RS:1;03a2ae042c95:41599 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T20:28:19,925 DEBUG [RS:2;03a2ae042c95:37285 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T20:28:19,925 DEBUG [RS:1;03a2ae042c95:41599 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T20:28:19,925 DEBUG [RS:2;03a2ae042c95:37285 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T20:28:19,926 DEBUG [RS:0;03a2ae042c95:34629 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T20:28:19,926 DEBUG [RS:0;03a2ae042c95:34629 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T20:28:19,940 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-10T20:28:19,945 DEBUG [RS:0;03a2ae042c95:34629 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T20:28:19,945 DEBUG [RS:2;03a2ae042c95:37285 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T20:28:19,945 DEBUG [RS:1;03a2ae042c95:41599 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T20:28:19,945 DEBUG [RS:0;03a2ae042c95:34629 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c61934a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=03a2ae042c95/172.17.0.3:0 2024-12-10T20:28:19,945 DEBUG [RS:1;03a2ae042c95:41599 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25519992, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=03a2ae042c95/172.17.0.3:0 2024-12-10T20:28:19,945 DEBUG [RS:2;03a2ae042c95:37285 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7168be4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=03a2ae042c95/172.17.0.3:0 2024-12-10T20:28:19,951 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-10T20:28:19,958 DEBUG [RS:2;03a2ae042c95:37285 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;03a2ae042c95:37285 2024-12-10T20:28:19,959 DEBUG [RS:1;03a2ae042c95:41599 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;03a2ae042c95:41599 2024-12-10T20:28:19,960 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-10T20:28:19,961 INFO [RS:2;03a2ae042c95:37285 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T20:28:19,961 INFO [RS:1;03a2ae042c95:41599 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T20:28:19,962 INFO [RS:2;03a2ae042c95:37285 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T20:28:19,962 INFO [RS:1;03a2ae042c95:41599 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T20:28:19,962 DEBUG [RS:2;03a2ae042c95:37285 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T20:28:19,962 DEBUG [RS:1;03a2ae042c95:41599 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T20:28:19,964 DEBUG [RS:0;03a2ae042c95:34629 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;03a2ae042c95:34629 2024-12-10T20:28:19,964 INFO [RS:1;03a2ae042c95:41599 {}] regionserver.HRegionServer(2659): reportForDuty to master=03a2ae042c95,34471,1733862497789 with port=41599, startcode=1733862498634 2024-12-10T20:28:19,964 INFO [RS:0;03a2ae042c95:34629 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T20:28:19,964 INFO [RS:0;03a2ae042c95:34629 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T20:28:19,964 DEBUG [RS:0;03a2ae042c95:34629 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T20:28:19,965 INFO [RS:2;03a2ae042c95:37285 {}] regionserver.HRegionServer(2659): reportForDuty to master=03a2ae042c95,34471,1733862497789 with port=37285, startcode=1733862498673 2024-12-10T20:28:19,966 INFO [RS:0;03a2ae042c95:34629 {}] regionserver.HRegionServer(2659): reportForDuty to master=03a2ae042c95,34471,1733862497789 with port=34629, startcode=1733862498540 2024-12-10T20:28:19,966 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 03a2ae042c95,34471,1733862497789 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-10T20:28:19,975 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/03a2ae042c95:0, corePoolSize=5, maxPoolSize=5 2024-12-10T20:28:19,975 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/03a2ae042c95:0, corePoolSize=5, maxPoolSize=5 2024-12-10T20:28:19,976 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/03a2ae042c95:0, corePoolSize=5, maxPoolSize=5 2024-12-10T20:28:19,976 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/03a2ae042c95:0, corePoolSize=5, maxPoolSize=5 2024-12-10T20:28:19,976 DEBUG [RS:0;03a2ae042c95:34629 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T20:28:19,976 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/03a2ae042c95:0, corePoolSize=10, maxPoolSize=10 2024-12-10T20:28:19,976 DEBUG [RS:1;03a2ae042c95:41599 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T20:28:19,976 DEBUG [RS:2;03a2ae042c95:37285 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T20:28:19,976 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:19,976 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/03a2ae042c95:0, corePoolSize=2, maxPoolSize=2 2024-12-10T20:28:19,977 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:19,981 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733862529981 2024-12-10T20:28:19,982 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T20:28:19,983 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-10T20:28:19,983 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-10T20:28:19,984 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-10T20:28:19,988 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-10T20:28:19,989 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-10T20:28:19,989 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-10T20:28:19,989 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-10T20:28:19,990 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:19,991 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-10T20:28:19,991 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:19,997 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-10T20:28:19,998 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-10T20:28:19,999 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-10T20:28:20,009 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-10T20:28:20,009 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-10T20:28:20,011 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/03a2ae042c95:0:becomeActiveMaster-HFileCleaner.large.0-1733862500010,5,FailOnTimeoutGroup] 2024-12-10T20:28:20,012 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/03a2ae042c95:0:becomeActiveMaster-HFileCleaner.small.0-1733862500011,5,FailOnTimeoutGroup] 2024-12-10T20:28:20,012 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,012 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-10T20:28:20,013 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,014 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741831_1007 (size=1321) 2024-12-10T20:28:20,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741831_1007 (size=1321) 2024-12-10T20:28:20,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741831_1007 (size=1321) 2024-12-10T20:28:20,054 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-10T20:28:20,055 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b 2024-12-10T20:28:20,057 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44303, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T20:28:20,057 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50391, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T20:28:20,057 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49353, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T20:28:20,063 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34471 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 03a2ae042c95,34629,1733862498540 2024-12-10T20:28:20,066 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34471 {}] master.ServerManager(517): Registering regionserver=03a2ae042c95,34629,1733862498540 2024-12-10T20:28:20,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741832_1008 (size=32) 2024-12-10T20:28:20,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741832_1008 (size=32) 2024-12-10T20:28:20,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741832_1008 (size=32) 2024-12-10T20:28:20,074 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:20,077 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-10T20:28:20,079 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34471 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 03a2ae042c95,37285,1733862498673 2024-12-10T20:28:20,079 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34471 {}] master.ServerManager(517): Registering regionserver=03a2ae042c95,37285,1733862498673 2024-12-10T20:28:20,081 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T20:28:20,081 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:20,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T20:28:20,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-10T20:28:20,084 DEBUG [RS:0;03a2ae042c95:34629 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b 2024-12-10T20:28:20,084 DEBUG [RS:0;03a2ae042c95:34629 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33493 2024-12-10T20:28:20,084 DEBUG [RS:0;03a2ae042c95:34629 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T20:28:20,086 DEBUG [RS:2;03a2ae042c95:37285 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b 2024-12-10T20:28:20,086 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34471 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 03a2ae042c95,41599,1733862498634 2024-12-10T20:28:20,086 DEBUG [RS:2;03a2ae042c95:37285 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33493 2024-12-10T20:28:20,086 DEBUG [RS:2;03a2ae042c95:37285 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T20:28:20,086 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34471 {}] master.ServerManager(517): Registering regionserver=03a2ae042c95,41599,1733862498634 2024-12-10T20:28:20,089 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T20:28:20,090 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:20,091 DEBUG [RS:1;03a2ae042c95:41599 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b 2024-12-10T20:28:20,091 DEBUG [RS:1;03a2ae042c95:41599 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33493 2024-12-10T20:28:20,091 DEBUG [RS:1;03a2ae042c95:41599 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T20:28:20,095 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T20:28:20,095 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-10T20:28:20,099 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T20:28:20,099 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:20,100 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T20:28:20,101 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-10T20:28:20,105 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T20:28:20,105 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:20,107 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T20:28:20,107 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T20:28:20,109 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740 2024-12-10T20:28:20,110 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740 2024-12-10T20:28:20,114 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T20:28:20,114 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T20:28:20,115 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T20:28:20,119 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T20:28:20,125 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:28:20,127 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75486900, jitterRate=0.12484246492385864}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T20:28:20,130 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733862500074Initializing all the Stores at 1733862500076 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733862500077 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733862500077Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862500077Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733862500077Cleaning up temporary data from old regions at 1733862500114 (+37 ms)Region opened successfully at 1733862500129 (+15 ms) 2024-12-10T20:28:20,130 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T20:28:20,130 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T20:28:20,130 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T20:28:20,130 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T20:28:20,130 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T20:28:20,133 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T20:28:20,133 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733862500130Disabling compacts and flushes for region at 1733862500130Disabling writes for close at 1733862500130Writing region close event to WAL at 1733862500132 (+2 ms)Closed at 1733862500133 (+1 ms) 2024-12-10T20:28:20,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T20:28:20,137 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T20:28:20,137 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-10T20:28:20,137 DEBUG [RS:2;03a2ae042c95:37285 {}] zookeeper.ZKUtil(111): regionserver:37285-0x100116a21c40003, quorum=127.0.0.1:53600, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/03a2ae042c95,37285,1733862498673 2024-12-10T20:28:20,137 WARN [RS:2;03a2ae042c95:37285 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T20:28:20,138 INFO [RS:2;03a2ae042c95:37285 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:20,138 DEBUG [RS:1;03a2ae042c95:41599 {}] zookeeper.ZKUtil(111): regionserver:41599-0x100116a21c40002, quorum=127.0.0.1:53600, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/03a2ae042c95,41599,1733862498634 2024-12-10T20:28:20,138 DEBUG [RS:0;03a2ae042c95:34629 {}] zookeeper.ZKUtil(111): regionserver:34629-0x100116a21c40001, quorum=127.0.0.1:53600, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/03a2ae042c95,34629,1733862498540 2024-12-10T20:28:20,138 WARN [RS:1;03a2ae042c95:41599 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T20:28:20,138 WARN [RS:0;03a2ae042c95:34629 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T20:28:20,138 INFO [RS:1;03a2ae042c95:41599 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:20,138 DEBUG [RS:2;03a2ae042c95:37285 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,37285,1733862498673 2024-12-10T20:28:20,138 INFO [RS:0;03a2ae042c95:34629 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:20,138 DEBUG [RS:1;03a2ae042c95:41599 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,41599,1733862498634 2024-12-10T20:28:20,138 DEBUG [RS:0;03a2ae042c95:34629 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,34629,1733862498540 2024-12-10T20:28:20,142 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [03a2ae042c95,34629,1733862498540] 2024-12-10T20:28:20,143 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [03a2ae042c95,37285,1733862498673] 2024-12-10T20:28:20,143 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [03a2ae042c95,41599,1733862498634] 2024-12-10T20:28:20,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T20:28:20,163 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T20:28:20,173 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-10T20:28:20,184 INFO [RS:2;03a2ae042c95:37285 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T20:28:20,184 INFO [RS:0;03a2ae042c95:34629 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T20:28:20,184 INFO [RS:1;03a2ae042c95:41599 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T20:28:20,197 INFO [RS:1;03a2ae042c95:41599 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T20:28:20,197 INFO [RS:0;03a2ae042c95:34629 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T20:28:20,197 INFO [RS:2;03a2ae042c95:37285 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T20:28:20,204 INFO [RS:0;03a2ae042c95:34629 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T20:28:20,204 INFO [RS:2;03a2ae042c95:37285 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T20:28:20,204 INFO [RS:1;03a2ae042c95:41599 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T20:28:20,204 INFO [RS:2;03a2ae042c95:37285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,204 INFO [RS:0;03a2ae042c95:34629 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,204 INFO [RS:1;03a2ae042c95:41599 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,206 INFO [RS:0;03a2ae042c95:34629 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T20:28:20,206 INFO [RS:1;03a2ae042c95:41599 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T20:28:20,210 INFO [RS:2;03a2ae042c95:37285 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T20:28:20,212 INFO [RS:0;03a2ae042c95:34629 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T20:28:20,212 INFO [RS:2;03a2ae042c95:37285 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T20:28:20,212 INFO [RS:1;03a2ae042c95:41599 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T20:28:20,214 INFO [RS:0;03a2ae042c95:34629 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,214 INFO [RS:2;03a2ae042c95:37285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,214 INFO [RS:1;03a2ae042c95:41599 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,214 DEBUG [RS:0;03a2ae042c95:34629 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,214 DEBUG [RS:2;03a2ae042c95:37285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,214 DEBUG [RS:1;03a2ae042c95:41599 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,214 DEBUG [RS:0;03a2ae042c95:34629 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,214 DEBUG [RS:2;03a2ae042c95:37285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,214 DEBUG [RS:1;03a2ae042c95:41599 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,214 DEBUG [RS:0;03a2ae042c95:34629 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,215 DEBUG [RS:0;03a2ae042c95:34629 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,215 DEBUG [RS:1;03a2ae042c95:41599 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,215 DEBUG [RS:2;03a2ae042c95:37285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,215 DEBUG [RS:0;03a2ae042c95:34629 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,215 DEBUG [RS:1;03a2ae042c95:41599 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,215 DEBUG [RS:2;03a2ae042c95:37285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,215 DEBUG [RS:0;03a2ae042c95:34629 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/03a2ae042c95:0, corePoolSize=2, maxPoolSize=2 2024-12-10T20:28:20,215 DEBUG [RS:1;03a2ae042c95:41599 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,215 DEBUG [RS:0;03a2ae042c95:34629 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,215 DEBUG [RS:2;03a2ae042c95:37285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,215 DEBUG [RS:1;03a2ae042c95:41599 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/03a2ae042c95:0, corePoolSize=2, maxPoolSize=2 2024-12-10T20:28:20,215 DEBUG [RS:0;03a2ae042c95:34629 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,215 DEBUG [RS:2;03a2ae042c95:37285 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/03a2ae042c95:0, corePoolSize=2, maxPoolSize=2 2024-12-10T20:28:20,215 DEBUG [RS:1;03a2ae042c95:41599 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,215 DEBUG [RS:0;03a2ae042c95:34629 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,215 DEBUG [RS:2;03a2ae042c95:37285 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,215 DEBUG [RS:1;03a2ae042c95:41599 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,215 DEBUG [RS:0;03a2ae042c95:34629 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,215 DEBUG [RS:2;03a2ae042c95:37285 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,215 DEBUG [RS:1;03a2ae042c95:41599 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,215 DEBUG [RS:0;03a2ae042c95:34629 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,215 DEBUG [RS:2;03a2ae042c95:37285 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,215 DEBUG [RS:1;03a2ae042c95:41599 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,216 DEBUG [RS:0;03a2ae042c95:34629 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,216 DEBUG [RS:2;03a2ae042c95:37285 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,216 DEBUG [RS:1;03a2ae042c95:41599 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,216 DEBUG [RS:0;03a2ae042c95:34629 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/03a2ae042c95:0, corePoolSize=3, maxPoolSize=3 2024-12-10T20:28:20,216 DEBUG [RS:1;03a2ae042c95:41599 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,216 DEBUG [RS:2;03a2ae042c95:37285 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,216 DEBUG [RS:0;03a2ae042c95:34629 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0, corePoolSize=3, maxPoolSize=3 2024-12-10T20:28:20,216 DEBUG [RS:1;03a2ae042c95:41599 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/03a2ae042c95:0, corePoolSize=3, maxPoolSize=3 2024-12-10T20:28:20,216 DEBUG [RS:2;03a2ae042c95:37285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/03a2ae042c95:0, corePoolSize=1, maxPoolSize=1 2024-12-10T20:28:20,216 DEBUG [RS:1;03a2ae042c95:41599 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0, corePoolSize=3, maxPoolSize=3 2024-12-10T20:28:20,216 DEBUG [RS:2;03a2ae042c95:37285 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/03a2ae042c95:0, corePoolSize=3, maxPoolSize=3 2024-12-10T20:28:20,216 DEBUG [RS:2;03a2ae042c95:37285 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/03a2ae042c95:0, corePoolSize=3, maxPoolSize=3 2024-12-10T20:28:20,222 INFO [RS:0;03a2ae042c95:34629 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,222 INFO [RS:0;03a2ae042c95:34629 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,222 INFO [RS:0;03a2ae042c95:34629 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,223 INFO [RS:0;03a2ae042c95:34629 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,223 INFO [RS:1;03a2ae042c95:41599 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,223 INFO [RS:1;03a2ae042c95:41599 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,223 INFO [RS:1;03a2ae042c95:41599 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,223 INFO [RS:0;03a2ae042c95:34629 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,223 INFO [RS:1;03a2ae042c95:41599 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,223 INFO [RS:0;03a2ae042c95:34629 {}] hbase.ChoreService(168): Chore ScheduledChore name=03a2ae042c95,34629,1733862498540-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T20:28:20,223 INFO [RS:1;03a2ae042c95:41599 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,223 INFO [RS:1;03a2ae042c95:41599 {}] hbase.ChoreService(168): Chore ScheduledChore name=03a2ae042c95,41599,1733862498634-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T20:28:20,224 INFO [RS:2;03a2ae042c95:37285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,225 INFO [RS:2;03a2ae042c95:37285 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,225 INFO [RS:2;03a2ae042c95:37285 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,225 INFO [RS:2;03a2ae042c95:37285 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,225 INFO [RS:2;03a2ae042c95:37285 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,225 INFO [RS:2;03a2ae042c95:37285 {}] hbase.ChoreService(168): Chore ScheduledChore name=03a2ae042c95,37285,1733862498673-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T20:28:20,247 INFO [RS:0;03a2ae042c95:34629 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T20:28:20,247 INFO [RS:1;03a2ae042c95:41599 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T20:28:20,248 INFO [RS:2;03a2ae042c95:37285 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T20:28:20,249 INFO [RS:0;03a2ae042c95:34629 {}] hbase.ChoreService(168): Chore ScheduledChore name=03a2ae042c95,34629,1733862498540-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,249 INFO [RS:2;03a2ae042c95:37285 {}] hbase.ChoreService(168): Chore ScheduledChore name=03a2ae042c95,37285,1733862498673-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,249 INFO [RS:1;03a2ae042c95:41599 {}] hbase.ChoreService(168): Chore ScheduledChore name=03a2ae042c95,41599,1733862498634-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,249 INFO [RS:2;03a2ae042c95:37285 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,250 INFO [RS:0;03a2ae042c95:34629 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,250 INFO [RS:1;03a2ae042c95:41599 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,250 INFO [RS:2;03a2ae042c95:37285 {}] regionserver.Replication(171): 03a2ae042c95,37285,1733862498673 started 2024-12-10T20:28:20,250 INFO [RS:1;03a2ae042c95:41599 {}] regionserver.Replication(171): 03a2ae042c95,41599,1733862498634 started 2024-12-10T20:28:20,250 INFO [RS:0;03a2ae042c95:34629 {}] regionserver.Replication(171): 03a2ae042c95,34629,1733862498540 started 2024-12-10T20:28:20,274 INFO [RS:2;03a2ae042c95:37285 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,274 INFO [RS:1;03a2ae042c95:41599 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,274 INFO [RS:0;03a2ae042c95:34629 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:20,274 INFO [RS:2;03a2ae042c95:37285 {}] regionserver.HRegionServer(1482): Serving as 03a2ae042c95,37285,1733862498673, RpcServer on 03a2ae042c95/172.17.0.3:37285, sessionid=0x100116a21c40003 2024-12-10T20:28:20,274 INFO [RS:0;03a2ae042c95:34629 {}] regionserver.HRegionServer(1482): Serving as 03a2ae042c95,34629,1733862498540, RpcServer on 03a2ae042c95/172.17.0.3:34629, sessionid=0x100116a21c40001 2024-12-10T20:28:20,274 INFO [RS:1;03a2ae042c95:41599 {}] regionserver.HRegionServer(1482): Serving as 03a2ae042c95,41599,1733862498634, RpcServer on 03a2ae042c95/172.17.0.3:41599, sessionid=0x100116a21c40002 2024-12-10T20:28:20,275 DEBUG [RS:2;03a2ae042c95:37285 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T20:28:20,275 DEBUG [RS:1;03a2ae042c95:41599 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T20:28:20,275 DEBUG [RS:0;03a2ae042c95:34629 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T20:28:20,276 DEBUG [RS:1;03a2ae042c95:41599 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 03a2ae042c95,41599,1733862498634 2024-12-10T20:28:20,276 DEBUG [RS:0;03a2ae042c95:34629 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 03a2ae042c95,34629,1733862498540 2024-12-10T20:28:20,276 DEBUG [RS:2;03a2ae042c95:37285 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 03a2ae042c95,37285,1733862498673 2024-12-10T20:28:20,276 DEBUG [RS:2;03a2ae042c95:37285 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '03a2ae042c95,37285,1733862498673' 2024-12-10T20:28:20,276 DEBUG [RS:0;03a2ae042c95:34629 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '03a2ae042c95,34629,1733862498540' 2024-12-10T20:28:20,276 DEBUG [RS:1;03a2ae042c95:41599 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '03a2ae042c95,41599,1733862498634' 2024-12-10T20:28:20,276 DEBUG [RS:2;03a2ae042c95:37285 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T20:28:20,276 DEBUG [RS:0;03a2ae042c95:34629 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T20:28:20,276 DEBUG [RS:1;03a2ae042c95:41599 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T20:28:20,277 DEBUG [RS:1;03a2ae042c95:41599 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T20:28:20,277 DEBUG [RS:2;03a2ae042c95:37285 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T20:28:20,277 DEBUG [RS:0;03a2ae042c95:34629 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T20:28:20,278 DEBUG [RS:1;03a2ae042c95:41599 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T20:28:20,278 DEBUG [RS:1;03a2ae042c95:41599 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T20:28:20,278 DEBUG [RS:1;03a2ae042c95:41599 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 03a2ae042c95,41599,1733862498634 2024-12-10T20:28:20,278 DEBUG [RS:0;03a2ae042c95:34629 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T20:28:20,279 DEBUG [RS:1;03a2ae042c95:41599 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '03a2ae042c95,41599,1733862498634' 2024-12-10T20:28:20,279 DEBUG [RS:0;03a2ae042c95:34629 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T20:28:20,279 DEBUG [RS:1;03a2ae042c95:41599 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T20:28:20,279 DEBUG [RS:0;03a2ae042c95:34629 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 03a2ae042c95,34629,1733862498540 2024-12-10T20:28:20,279 DEBUG [RS:0;03a2ae042c95:34629 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '03a2ae042c95,34629,1733862498540' 2024-12-10T20:28:20,279 DEBUG [RS:0;03a2ae042c95:34629 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T20:28:20,279 DEBUG [RS:2;03a2ae042c95:37285 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T20:28:20,279 DEBUG [RS:2;03a2ae042c95:37285 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T20:28:20,279 DEBUG [RS:2;03a2ae042c95:37285 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 03a2ae042c95,37285,1733862498673 2024-12-10T20:28:20,279 DEBUG [RS:2;03a2ae042c95:37285 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '03a2ae042c95,37285,1733862498673' 2024-12-10T20:28:20,279 DEBUG [RS:2;03a2ae042c95:37285 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T20:28:20,279 DEBUG [RS:1;03a2ae042c95:41599 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T20:28:20,279 DEBUG [RS:0;03a2ae042c95:34629 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T20:28:20,280 DEBUG [RS:2;03a2ae042c95:37285 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T20:28:20,280 DEBUG [RS:1;03a2ae042c95:41599 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T20:28:20,280 DEBUG [RS:0;03a2ae042c95:34629 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T20:28:20,280 INFO [RS:1;03a2ae042c95:41599 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T20:28:20,280 INFO [RS:0;03a2ae042c95:34629 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T20:28:20,280 INFO [RS:1;03a2ae042c95:41599 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T20:28:20,280 INFO [RS:0;03a2ae042c95:34629 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T20:28:20,280 DEBUG [RS:2;03a2ae042c95:37285 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T20:28:20,280 INFO [RS:2;03a2ae042c95:37285 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T20:28:20,280 INFO [RS:2;03a2ae042c95:37285 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T20:28:20,324 WARN [03a2ae042c95:34471 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-10T20:28:20,385 INFO [RS:1;03a2ae042c95:41599 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T20:28:20,385 INFO [RS:2;03a2ae042c95:37285 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T20:28:20,385 INFO [RS:0;03a2ae042c95:34629 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T20:28:20,388 INFO [RS:0;03a2ae042c95:34629 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=03a2ae042c95%2C34629%2C1733862498540, suffix=, logDir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,34629,1733862498540, archiveDir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/oldWALs, maxLogs=32 2024-12-10T20:28:20,388 INFO [RS:1;03a2ae042c95:41599 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=03a2ae042c95%2C41599%2C1733862498634, suffix=, logDir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,41599,1733862498634, archiveDir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/oldWALs, maxLogs=32 2024-12-10T20:28:20,388 INFO [RS:2;03a2ae042c95:37285 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=03a2ae042c95%2C37285%2C1733862498673, suffix=, logDir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,37285,1733862498673, archiveDir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/oldWALs, maxLogs=32 2024-12-10T20:28:20,410 DEBUG [RS:2;03a2ae042c95:37285 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,37285,1733862498673/03a2ae042c95%2C37285%2C1733862498673.1733862500392, exclude list is [], retry=0 2024-12-10T20:28:20,412 DEBUG [RS:1;03a2ae042c95:41599 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,41599,1733862498634/03a2ae042c95%2C41599%2C1733862498634.1733862500392, exclude list is [], retry=0 2024-12-10T20:28:20,417 DEBUG [RS:0;03a2ae042c95:34629 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,34629,1733862498540/03a2ae042c95%2C34629%2C1733862498540.1733862500392, exclude list is [], retry=0 2024-12-10T20:28:20,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:20,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:20,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:20,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:20,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:20,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:20,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:20,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:20,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:20,444 INFO [RS:2;03a2ae042c95:37285 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,37285,1733862498673/03a2ae042c95%2C37285%2C1733862498673.1733862500392 2024-12-10T20:28:20,453 DEBUG [RS:2;03a2ae042c95:37285 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:40461:40461)] 2024-12-10T20:28:20,460 INFO [RS:1;03a2ae042c95:41599 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,41599,1733862498634/03a2ae042c95%2C41599%2C1733862498634.1733862500392 2024-12-10T20:28:20,464 INFO [RS:0;03a2ae042c95:34629 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,34629,1733862498540/03a2ae042c95%2C34629%2C1733862498540.1733862500392 2024-12-10T20:28:20,469 DEBUG [RS:1;03a2ae042c95:41599 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:40461:40461)] 2024-12-10T20:28:20,471 DEBUG [RS:0;03a2ae042c95:34629 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:46745:46745)] 2024-12-10T20:28:20,578 DEBUG [03a2ae042c95:34471 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-10T20:28:20,588 DEBUG [03a2ae042c95:34471 {}] balancer.BalancerClusterState(204): Hosts are {03a2ae042c95=0} racks are {/default-rack=0} 2024-12-10T20:28:20,594 DEBUG [03a2ae042c95:34471 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T20:28:20,594 DEBUG [03a2ae042c95:34471 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T20:28:20,594 DEBUG [03a2ae042c95:34471 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-10T20:28:20,595 DEBUG [03a2ae042c95:34471 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T20:28:20,595 DEBUG [03a2ae042c95:34471 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T20:28:20,595 DEBUG [03a2ae042c95:34471 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-10T20:28:20,595 INFO [03a2ae042c95:34471 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T20:28:20,595 INFO [03a2ae042c95:34471 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T20:28:20,595 INFO [03a2ae042c95:34471 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-10T20:28:20,595 DEBUG [03a2ae042c95:34471 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T20:28:20,601 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=03a2ae042c95,34629,1733862498540 2024-12-10T20:28:20,606 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 03a2ae042c95,34629,1733862498540, state=OPENING 2024-12-10T20:28:20,660 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-10T20:28:20,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:20,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37285-0x100116a21c40003, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:20,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41599-0x100116a21c40002, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:20,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34629-0x100116a21c40001, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:20,671 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T20:28:20,671 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T20:28:20,671 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T20:28:20,671 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T20:28:20,673 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T20:28:20,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=03a2ae042c95,34629,1733862498540}] 2024-12-10T20:28:20,857 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T20:28:20,859 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59739, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T20:28:20,871 INFO [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-10T20:28:20,871 INFO [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:20,872 INFO [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T20:28:20,876 INFO [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=03a2ae042c95%2C34629%2C1733862498540.meta, suffix=.meta, logDir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,34629,1733862498540, archiveDir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/oldWALs, maxLogs=32 2024-12-10T20:28:20,890 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,34629,1733862498540/03a2ae042c95%2C34629%2C1733862498540.meta.1733862500878.meta, exclude list is [], retry=0 2024-12-10T20:28:20,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:20,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:20,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:20,903 INFO [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,34629,1733862498540/03a2ae042c95%2C34629%2C1733862498540.meta.1733862500878.meta 2024-12-10T20:28:20,905 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:33825:33825)] 2024-12-10T20:28:20,905 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:20,906 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T20:28:20,909 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T20:28:20,913 INFO [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T20:28:20,917 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T20:28:20,918 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:20,918 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-10T20:28:20,918 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-10T20:28:20,921 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-10T20:28:20,922 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T20:28:20,922 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:20,923 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T20:28:20,923 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-10T20:28:20,924 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T20:28:20,924 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:20,925 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T20:28:20,925 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-10T20:28:20,926 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T20:28:20,926 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:20,927 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T20:28:20,927 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-10T20:28:20,929 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T20:28:20,929 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:20,930 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T20:28:20,930 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T20:28:20,931 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740 2024-12-10T20:28:20,934 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740 2024-12-10T20:28:20,936 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T20:28:20,936 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T20:28:20,937 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T20:28:20,939 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T20:28:20,940 INFO [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70245548, jitterRate=0.04674023389816284}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T20:28:20,941 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-10T20:28:20,942 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733862500919Writing region info on filesystem at 1733862500919Initializing all the Stores at 1733862500920 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733862500920Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733862500920Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862500921 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733862500921Cleaning up temporary data from old regions at 1733862500936 (+15 ms)Running coprocessor post-open hooks at 1733862500941 (+5 ms)Region opened successfully at 1733862500942 (+1 ms) 2024-12-10T20:28:20,947 INFO [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733862500847 2024-12-10T20:28:20,957 DEBUG [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T20:28:20,957 INFO [RS_OPEN_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-10T20:28:20,959 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=03a2ae042c95,34629,1733862498540 2024-12-10T20:28:20,961 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 03a2ae042c95,34629,1733862498540, state=OPEN 2024-12-10T20:28:20,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37285-0x100116a21c40003, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T20:28:20,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T20:28:20,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34629-0x100116a21c40001, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T20:28:20,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41599-0x100116a21c40002, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T20:28:20,974 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T20:28:20,974 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T20:28:20,974 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T20:28:20,974 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T20:28:20,974 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=03a2ae042c95,34629,1733862498540 2024-12-10T20:28:20,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-10T20:28:20,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=03a2ae042c95,34629,1733862498540 in 299 msec 2024-12-10T20:28:20,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-10T20:28:20,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 838 msec 2024-12-10T20:28:20,990 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T20:28:20,990 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-10T20:28:21,035 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:28:21,037 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,34629,1733862498540, seqNum=-1] 2024-12-10T20:28:21,065 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:28:21,067 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58285, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:28:21,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1970 sec 2024-12-10T20:28:21,092 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733862501091, completionTime=-1 2024-12-10T20:28:21,094 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-10T20:28:21,094 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-10T20:28:21,122 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-10T20:28:21,122 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733862561122 2024-12-10T20:28:21,122 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733862621122 2024-12-10T20:28:21,122 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-12-10T20:28:21,124 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-10T20:28:21,132 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=03a2ae042c95,34471,1733862497789-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:21,132 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=03a2ae042c95,34471,1733862497789-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:21,133 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=03a2ae042c95,34471,1733862497789-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:21,135 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-03a2ae042c95:34471, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:21,138 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:21,139 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:21,142 DEBUG [master/03a2ae042c95:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-10T20:28:21,175 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.204sec 2024-12-10T20:28:21,177 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-10T20:28:21,179 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-10T20:28:21,180 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-10T20:28:21,181 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-10T20:28:21,181 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-10T20:28:21,182 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=03a2ae042c95,34471,1733862497789-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T20:28:21,183 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=03a2ae042c95,34471,1733862497789-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-10T20:28:21,189 DEBUG [master/03a2ae042c95:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-10T20:28:21,190 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-10T20:28:21,190 INFO [master/03a2ae042c95:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=03a2ae042c95,34471,1733862497789-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:21,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ddf6023, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:28:21,240 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 03a2ae042c95,34471,-1 for getting cluster id 2024-12-10T20:28:21,243 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T20:28:21,256 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '534d3ae7-c7ac-4fbf-af98-6ae9e35d04c4' 2024-12-10T20:28:21,259 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T20:28:21,259 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "534d3ae7-c7ac-4fbf-af98-6ae9e35d04c4" 2024-12-10T20:28:21,260 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5231620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:28:21,260 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [03a2ae042c95,34471,-1] 2024-12-10T20:28:21,263 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T20:28:21,265 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:28:21,267 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T20:28:21,271 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65c83e31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T20:28:21,271 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T20:28:21,280 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=03a2ae042c95,34629,1733862498540, seqNum=-1] 2024-12-10T20:28:21,281 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:28:21,284 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:28:21,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=03a2ae042c95,34471,1733862497789 2024-12-10T20:28:21,310 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:33493/hbase 2024-12-10T20:28:21,333 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=365, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=272, ProcessCount=11, AvailableMemoryMB=5468 2024-12-10T20:28:21,360 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:21,364 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:21,366 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T20:28:21,372 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-17265219, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/hregion-17265219, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:21,395 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-17265219/hregion-17265219.1733862501374, exclude list is [], retry=0 2024-12-10T20:28:21,402 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:21,406 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:21,411 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:21,426 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-17265219/hregion-17265219.1733862501374 2024-12-10T20:28:21,426 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:33825:33825)] 2024-12-10T20:28:21,426 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2c7ebc40f7602dccabb7d1b9bb23f125, NAME => 'testReplayEditsWrittenIntoWAL,,1733862501361.2c7ebc40f7602dccabb7d1b9bb23f125.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33493/hbase 2024-12-10T20:28:21,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741838_1014 (size=64) 2024-12-10T20:28:21,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741838_1014 (size=64) 2024-12-10T20:28:21,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741838_1014 (size=64) 2024-12-10T20:28:21,456 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733862501361.2c7ebc40f7602dccabb7d1b9bb23f125.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:21,458 INFO [StoreOpener-2c7ebc40f7602dccabb7d1b9bb23f125-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2c7ebc40f7602dccabb7d1b9bb23f125 2024-12-10T20:28:21,462 INFO [StoreOpener-2c7ebc40f7602dccabb7d1b9bb23f125-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c7ebc40f7602dccabb7d1b9bb23f125 columnFamilyName a 2024-12-10T20:28:21,462 DEBUG [StoreOpener-2c7ebc40f7602dccabb7d1b9bb23f125-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:21,463 INFO [StoreOpener-2c7ebc40f7602dccabb7d1b9bb23f125-1 {}] regionserver.HStore(327): Store=2c7ebc40f7602dccabb7d1b9bb23f125/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:21,463 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2c7ebc40f7602dccabb7d1b9bb23f125 2024-12-10T20:28:21,468 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125 2024-12-10T20:28:21,473 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125 2024-12-10T20:28:21,475 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2c7ebc40f7602dccabb7d1b9bb23f125 2024-12-10T20:28:21,475 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2c7ebc40f7602dccabb7d1b9bb23f125 2024-12-10T20:28:21,480 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2c7ebc40f7602dccabb7d1b9bb23f125 2024-12-10T20:28:21,489 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:28:21,490 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2c7ebc40f7602dccabb7d1b9bb23f125; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70815342, jitterRate=0.05523082613945007}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T20:28:21,491 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2c7ebc40f7602dccabb7d1b9bb23f125: Writing region info on filesystem at 1733862501456Initializing all the Stores at 1733862501458 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862501458Cleaning up temporary data from old regions at 1733862501475 (+17 ms)Region opened successfully at 1733862501491 (+16 ms) 2024-12-10T20:28:21,492 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2c7ebc40f7602dccabb7d1b9bb23f125, disabling compactions & flushes 2024-12-10T20:28:21,492 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733862501361.2c7ebc40f7602dccabb7d1b9bb23f125. 2024-12-10T20:28:21,492 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733862501361.2c7ebc40f7602dccabb7d1b9bb23f125. 2024-12-10T20:28:21,492 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733862501361.2c7ebc40f7602dccabb7d1b9bb23f125. after waiting 0 ms 2024-12-10T20:28:21,492 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733862501361.2c7ebc40f7602dccabb7d1b9bb23f125. 2024-12-10T20:28:21,493 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733862501361.2c7ebc40f7602dccabb7d1b9bb23f125. 2024-12-10T20:28:21,493 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2c7ebc40f7602dccabb7d1b9bb23f125: Waiting for close lock at 1733862501492Disabling compacts and flushes for region at 1733862501492Disabling writes for close at 1733862501492Writing region close event to WAL at 1733862501493 (+1 ms)Closed at 1733862501493 2024-12-10T20:28:21,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741837_1013 (size=93) 2024-12-10T20:28:21,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741837_1013 (size=93) 2024-12-10T20:28:21,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741837_1013 (size=93) 2024-12-10T20:28:21,517 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T20:28:21,518 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-17265219:(num 1733862501374) 2024-12-10T20:28:21,520 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-10T20:28:21,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741839_1015 (size=276) 2024-12-10T20:28:21,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741839_1015 (size=276) 2024-12-10T20:28:21,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741839_1015 (size=276) 2024-12-10T20:28:21,545 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-10T20:28:21,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741840_1016 (size=230) 2024-12-10T20:28:21,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741840_1016 (size=230) 2024-12-10T20:28:21,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741840_1016 (size=230) 2024-12-10T20:28:21,575 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-1, size=276 (276bytes) 2024-12-10T20:28:21,576 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-10T20:28:21,576 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-10T20:28:21,576 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-1 2024-12-10T20:28:21,581 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-1 after 3ms 2024-12-10T20:28:21,586 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:21,587 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-1 took 13ms 2024-12-10T20:28:21,590 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-1 so closing down 2024-12-10T20:28:21,593 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-10T20:28:21,594 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T20:28:21,596 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/recovered.edits/0000000000000000001-wal-1.temp 2024-12-10T20:28:21,597 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T20:28:21,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741841_1017 (size=276) 2024-12-10T20:28:21,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741841_1017 (size=276) 2024-12-10T20:28:21,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741841_1017 (size=276) 2024-12-10T20:28:21,608 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-10T20:28:21,611 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/recovered.edits/0000000000000000002 2024-12-10T20:28:21,615 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-12-10T20:28:21,615 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-1, journal: Splitting hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-1, size=276 (276bytes) at 1733862501575Finishing writing output for hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-1 so closing down at 1733862501590 (+15 ms)Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/recovered.edits/0000000000000000001-wal-1.temp at 1733862501596 (+6 ms)3 split writer threads finished at 1733862501598 (+2 ms)Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733862501608 (+10 ms)Rename recovered edits hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/recovered.edits/0000000000000000002 at 1733862501611 (+3 ms)Processed 2 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1733862501615 (+4 ms) 2024-12-10T20:28:21,631 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-2, size=230 (230bytes) 2024-12-10T20:28:21,631 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-2 2024-12-10T20:28:21,632 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-2 after 1ms 2024-12-10T20:28:21,635 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:21,635 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-2 took 4ms 2024-12-10T20:28:21,638 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-2 so closing down 2024-12-10T20:28:21,638 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T20:28:21,640 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-10T20:28:21,642 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/recovered.edits/0000000000000000002-wal-2.temp 2024-12-10T20:28:21,642 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T20:28:21,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741842_1018 (size=230) 2024-12-10T20:28:21,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741842_1018 (size=230) 2024-12-10T20:28:21,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741842_1018 (size=230) 2024-12-10T20:28:21,651 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-10T20:28:21,655 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:21,657 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/recovered.edits/0000000000000000002-wal-2.temp, length=230 2024-12-10T20:28:21,659 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-12-10T20:28:21,659 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-2, journal: Splitting hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-2, size=230 (230bytes) at 1733862501631Finishing writing output for hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-2 so closing down at 1733862501638 (+7 ms)Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/recovered.edits/0000000000000000002-wal-2.temp at 1733862501642 (+4 ms)3 split writer threads finished at 1733862501642Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733862501651 (+9 ms)Processed 1 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1733862501659 (+8 ms) 2024-12-10T20:28:21,659 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T20:28:21,662 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:21,673 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal.1733862501662, exclude list is [], retry=0 2024-12-10T20:28:21,677 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:21,678 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:21,678 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:21,682 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal.1733862501662 2024-12-10T20:28:21,682 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:46745:46745)] 2024-12-10T20:28:21,683 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2c7ebc40f7602dccabb7d1b9bb23f125, NAME => 'testReplayEditsWrittenIntoWAL,,1733862501361.2c7ebc40f7602dccabb7d1b9bb23f125.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:21,683 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733862501361.2c7ebc40f7602dccabb7d1b9bb23f125.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:21,683 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2c7ebc40f7602dccabb7d1b9bb23f125 2024-12-10T20:28:21,683 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2c7ebc40f7602dccabb7d1b9bb23f125 2024-12-10T20:28:21,686 INFO [StoreOpener-2c7ebc40f7602dccabb7d1b9bb23f125-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2c7ebc40f7602dccabb7d1b9bb23f125 2024-12-10T20:28:21,687 INFO [StoreOpener-2c7ebc40f7602dccabb7d1b9bb23f125-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c7ebc40f7602dccabb7d1b9bb23f125 columnFamilyName a 2024-12-10T20:28:21,687 DEBUG [StoreOpener-2c7ebc40f7602dccabb7d1b9bb23f125-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:21,689 INFO [StoreOpener-2c7ebc40f7602dccabb7d1b9bb23f125-1 {}] regionserver.HStore(327): Store=2c7ebc40f7602dccabb7d1b9bb23f125/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:21,689 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2c7ebc40f7602dccabb7d1b9bb23f125 2024-12-10T20:28:21,690 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125 2024-12-10T20:28:21,693 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125 2024-12-10T20:28:21,695 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/recovered.edits/0000000000000000002 2024-12-10T20:28:21,699 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:21,705 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/recovered.edits/0000000000000000002 2024-12-10T20:28:21,710 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2c7ebc40f7602dccabb7d1b9bb23f125 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-10T20:28:21,783 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/.tmp/a/d2fae12586054cd180850dedc76683d0 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733862501518/Put/seqid=0 2024-12-10T20:28:21,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741844_1020 (size=5170) 2024-12-10T20:28:21,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741844_1020 (size=5170) 2024-12-10T20:28:21,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741844_1020 (size=5170) 2024-12-10T20:28:21,809 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/.tmp/a/d2fae12586054cd180850dedc76683d0 2024-12-10T20:28:21,859 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/.tmp/a/d2fae12586054cd180850dedc76683d0 as hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/a/d2fae12586054cd180850dedc76683d0 2024-12-10T20:28:21,870 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/a/d2fae12586054cd180850dedc76683d0, entries=2, sequenceid=2, filesize=5.0 K 2024-12-10T20:28:21,877 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 2c7ebc40f7602dccabb7d1b9bb23f125 in 166ms, sequenceid=2, compaction requested=false; wal=null 2024-12-10T20:28:21,879 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/recovered.edits/0000000000000000002 2024-12-10T20:28:21,880 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2c7ebc40f7602dccabb7d1b9bb23f125 2024-12-10T20:28:21,880 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2c7ebc40f7602dccabb7d1b9bb23f125 2024-12-10T20:28:21,884 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2c7ebc40f7602dccabb7d1b9bb23f125 2024-12-10T20:28:21,889 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/2c7ebc40f7602dccabb7d1b9bb23f125/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-10T20:28:21,891 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2c7ebc40f7602dccabb7d1b9bb23f125; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62129803, jitterRate=-0.0741937905550003}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T20:28:21,892 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2c7ebc40f7602dccabb7d1b9bb23f125: Writing region info on filesystem at 1733862501683Initializing all the Stores at 1733862501685 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862501685Obtaining lock to block concurrent updates at 1733862501710 (+25 ms)Preparing flush snapshotting stores in 2c7ebc40f7602dccabb7d1b9bb23f125 at 1733862501710Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733862501361.2c7ebc40f7602dccabb7d1b9bb23f125., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733862501717 (+7 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733862501361.2c7ebc40f7602dccabb7d1b9bb23f125. at 1733862501717Flushing 2c7ebc40f7602dccabb7d1b9bb23f125/a: creating writer at 1733862501719 (+2 ms)Flushing 2c7ebc40f7602dccabb7d1b9bb23f125/a: appending metadata at 1733862501769 (+50 ms)Flushing 2c7ebc40f7602dccabb7d1b9bb23f125/a: closing flushed file at 1733862501773 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7864b056: reopening flushed file at 1733862501857 (+84 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 2c7ebc40f7602dccabb7d1b9bb23f125 in 166ms, sequenceid=2, compaction requested=false; wal=null at 1733862501877 (+20 ms)Cleaning up temporary data from old regions at 1733862501880 (+3 ms)Region opened successfully at 1733862501892 (+12 ms) 2024-12-10T20:28:21,919 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=377 (was 365) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:60544 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-76190982-172.17.0.3-1733862492788:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:33493/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:44044 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:51864 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-76190982-172.17.0.3-1733862492788:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:41660 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:43558 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:51896 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-76190982-172.17.0.3-1733862492788:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=272 (was 272), ProcessCount=11 (was 11), AvailableMemoryMB=5449 (was 5468) 2024-12-10T20:28:21,931 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=377, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=272, ProcessCount=11, AvailableMemoryMB=5448 2024-12-10T20:28:21,947 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:21,949 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:21,950 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T20:28:21,954 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-21051573, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/hregion-21051573, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:21,968 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-21051573/hregion-21051573.1733862501955, exclude list is [], retry=0 2024-12-10T20:28:21,972 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:21,973 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:21,977 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:21,981 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-21051573/hregion-21051573.1733862501955 2024-12-10T20:28:21,982 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:46745:46745)] 2024-12-10T20:28:21,982 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 167d47117c60cc19abe42487fbad794c, NAME => 'testReplayEditsWrittenIntoWAL,,1733862501948.167d47117c60cc19abe42487fbad794c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33493/hbase 2024-12-10T20:28:21,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741846_1022 (size=64) 2024-12-10T20:28:21,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741846_1022 (size=64) 2024-12-10T20:28:21,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741846_1022 (size=64) 2024-12-10T20:28:21,999 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733862501948.167d47117c60cc19abe42487fbad794c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:22,001 INFO [StoreOpener-167d47117c60cc19abe42487fbad794c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 167d47117c60cc19abe42487fbad794c 2024-12-10T20:28:22,003 INFO [StoreOpener-167d47117c60cc19abe42487fbad794c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 167d47117c60cc19abe42487fbad794c columnFamilyName a 2024-12-10T20:28:22,004 DEBUG [StoreOpener-167d47117c60cc19abe42487fbad794c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:22,004 INFO [StoreOpener-167d47117c60cc19abe42487fbad794c-1 {}] regionserver.HStore(327): Store=167d47117c60cc19abe42487fbad794c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:22,005 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 167d47117c60cc19abe42487fbad794c 2024-12-10T20:28:22,006 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c 2024-12-10T20:28:22,006 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c 2024-12-10T20:28:22,007 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 167d47117c60cc19abe42487fbad794c 2024-12-10T20:28:22,007 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 167d47117c60cc19abe42487fbad794c 2024-12-10T20:28:22,009 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 167d47117c60cc19abe42487fbad794c 2024-12-10T20:28:22,013 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:28:22,014 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 167d47117c60cc19abe42487fbad794c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58931758, jitterRate=-0.12184837460517883}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T20:28:22,014 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 167d47117c60cc19abe42487fbad794c: Writing region info on filesystem at 1733862501999Initializing all the Stores at 1733862502000 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862502000Cleaning up temporary data from old regions at 1733862502007 (+7 ms)Region opened successfully at 1733862502014 (+7 ms) 2024-12-10T20:28:22,015 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 167d47117c60cc19abe42487fbad794c, disabling compactions & flushes 2024-12-10T20:28:22,015 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733862501948.167d47117c60cc19abe42487fbad794c. 2024-12-10T20:28:22,015 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733862501948.167d47117c60cc19abe42487fbad794c. 2024-12-10T20:28:22,015 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733862501948.167d47117c60cc19abe42487fbad794c. after waiting 0 ms 2024-12-10T20:28:22,015 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733862501948.167d47117c60cc19abe42487fbad794c. 2024-12-10T20:28:22,015 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733862501948.167d47117c60cc19abe42487fbad794c. 2024-12-10T20:28:22,015 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 167d47117c60cc19abe42487fbad794c: Waiting for close lock at 1733862502014Disabling compacts and flushes for region at 1733862502014Disabling writes for close at 1733862502015 (+1 ms)Writing region close event to WAL at 1733862502015Closed at 1733862502015 2024-12-10T20:28:22,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741845_1021 (size=93) 2024-12-10T20:28:22,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741845_1021 (size=93) 2024-12-10T20:28:22,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741845_1021 (size=93) 2024-12-10T20:28:22,023 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T20:28:22,023 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-21051573:(num 1733862501955) 2024-12-10T20:28:22,025 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-10T20:28:22,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741847_1023 (size=276) 2024-12-10T20:28:22,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741847_1023 (size=276) 2024-12-10T20:28:22,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741847_1023 (size=276) 2024-12-10T20:28:22,445 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-10T20:28:22,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741848_1024 (size=230) 2024-12-10T20:28:22,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741848_1024 (size=230) 2024-12-10T20:28:22,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741848_1024 (size=230) 2024-12-10T20:28:22,476 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-2, size=230 (230bytes) 2024-12-10T20:28:22,477 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-2 2024-12-10T20:28:22,478 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-2 after 0ms 2024-12-10T20:28:22,481 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:22,481 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-2 took 5ms 2024-12-10T20:28:22,484 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-2 so closing down 2024-12-10T20:28:22,484 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T20:28:22,486 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-10T20:28:22,488 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000002-wal-2.temp 2024-12-10T20:28:22,488 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T20:28:22,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741849_1025 (size=230) 2024-12-10T20:28:22,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741849_1025 (size=230) 2024-12-10T20:28:22,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741849_1025 (size=230) 2024-12-10T20:28:22,496 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-10T20:28:22,498 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000002 2024-12-10T20:28:22,498 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-12-10T20:28:22,498 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-2, journal: Splitting hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-2, size=230 (230bytes) at 1733862502477Finishing writing output for hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-2 so closing down at 1733862502484 (+7 ms)Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000002-wal-2.temp at 1733862502488 (+4 ms)3 split writer threads finished at 1733862502488Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733862502496 (+8 ms)Rename recovered edits hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000002 at 1733862502498 (+2 ms)Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1733862502498 2024-12-10T20:28:22,515 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-1, size=276 (276bytes) 2024-12-10T20:28:22,515 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-1 2024-12-10T20:28:22,516 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-1 after 1ms 2024-12-10T20:28:22,519 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:22,520 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-1 took 5ms 2024-12-10T20:28:22,522 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-1 so closing down 2024-12-10T20:28:22,522 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T20:28:22,524 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-10T20:28:22,526 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000001-wal-1.temp 2024-12-10T20:28:22,526 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T20:28:22,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741850_1026 (size=276) 2024-12-10T20:28:22,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741850_1026 (size=276) 2024-12-10T20:28:22,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741850_1026 (size=276) 2024-12-10T20:28:22,535 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-10T20:28:22,538 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:22,540 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000002, length=230 2024-12-10T20:28:22,542 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000002 2024-12-10T20:28:22,542 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 22 ms; skipped=0; WAL=hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-12-10T20:28:22,543 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-1, journal: Splitting hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-1, size=276 (276bytes) at 1733862502515Finishing writing output for hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-1 so closing down at 1733862502522 (+7 ms)Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000001-wal-1.temp at 1733862502526 (+4 ms)3 split writer threads finished at 1733862502526Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733862502535 (+9 ms)Rename recovered edits hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000002 at 1733862502542 (+7 ms)Processed 2 edits across 1 Regions in 22 ms; skipped=0; WAL=hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1733862502543 (+1 ms) 2024-12-10T20:28:22,543 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T20:28:22,545 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:22,557 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal.1733862502545, exclude list is [], retry=0 2024-12-10T20:28:22,560 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:22,560 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:22,560 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:22,562 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal.1733862502545 2024-12-10T20:28:22,562 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:46745:46745)] 2024-12-10T20:28:22,563 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 167d47117c60cc19abe42487fbad794c, NAME => 'testReplayEditsWrittenIntoWAL,,1733862501948.167d47117c60cc19abe42487fbad794c.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:22,563 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733862501948.167d47117c60cc19abe42487fbad794c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:22,563 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 167d47117c60cc19abe42487fbad794c 2024-12-10T20:28:22,563 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 167d47117c60cc19abe42487fbad794c 2024-12-10T20:28:22,565 INFO [StoreOpener-167d47117c60cc19abe42487fbad794c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 167d47117c60cc19abe42487fbad794c 2024-12-10T20:28:22,566 INFO [StoreOpener-167d47117c60cc19abe42487fbad794c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 167d47117c60cc19abe42487fbad794c columnFamilyName a 2024-12-10T20:28:22,566 DEBUG [StoreOpener-167d47117c60cc19abe42487fbad794c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:22,567 INFO [StoreOpener-167d47117c60cc19abe42487fbad794c-1 {}] regionserver.HStore(327): Store=167d47117c60cc19abe42487fbad794c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:22,567 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 167d47117c60cc19abe42487fbad794c 2024-12-10T20:28:22,568 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c 2024-12-10T20:28:22,570 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c 2024-12-10T20:28:22,571 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000002 2024-12-10T20:28:22,573 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:22,575 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000002 2024-12-10T20:28:22,575 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 167d47117c60cc19abe42487fbad794c 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-10T20:28:22,589 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/.tmp/a/0048f05980584934a0eb64c0020f7717 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733862502023/Put/seqid=0 2024-12-10T20:28:22,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741852_1028 (size=5170) 2024-12-10T20:28:22,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741852_1028 (size=5170) 2024-12-10T20:28:22,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741852_1028 (size=5170) 2024-12-10T20:28:22,598 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/.tmp/a/0048f05980584934a0eb64c0020f7717 2024-12-10T20:28:22,608 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/.tmp/a/0048f05980584934a0eb64c0020f7717 as hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/a/0048f05980584934a0eb64c0020f7717 2024-12-10T20:28:22,619 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/a/0048f05980584934a0eb64c0020f7717, entries=2, sequenceid=2, filesize=5.0 K 2024-12-10T20:28:22,619 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 167d47117c60cc19abe42487fbad794c in 44ms, sequenceid=2, compaction requested=false; wal=null 2024-12-10T20:28:22,620 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/0000000000000000002 2024-12-10T20:28:22,621 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 167d47117c60cc19abe42487fbad794c 2024-12-10T20:28:22,621 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 167d47117c60cc19abe42487fbad794c 2024-12-10T20:28:22,625 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 167d47117c60cc19abe42487fbad794c 2024-12-10T20:28:22,630 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/167d47117c60cc19abe42487fbad794c/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-10T20:28:22,632 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 167d47117c60cc19abe42487fbad794c; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61977361, jitterRate=-0.07646535336971283}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T20:28:22,632 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 167d47117c60cc19abe42487fbad794c: Writing region info on filesystem at 1733862502563Initializing all the Stores at 1733862502564 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862502564Obtaining lock to block concurrent updates at 1733862502575 (+11 ms)Preparing flush snapshotting stores in 167d47117c60cc19abe42487fbad794c at 1733862502575Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733862501948.167d47117c60cc19abe42487fbad794c., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733862502575Flushing stores of testReplayEditsWrittenIntoWAL,,1733862501948.167d47117c60cc19abe42487fbad794c. at 1733862502575Flushing 167d47117c60cc19abe42487fbad794c/a: creating writer at 1733862502575Flushing 167d47117c60cc19abe42487fbad794c/a: appending metadata at 1733862502588 (+13 ms)Flushing 167d47117c60cc19abe42487fbad794c/a: closing flushed file at 1733862502588Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@899f4d5: reopening flushed file at 1733862502607 (+19 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 167d47117c60cc19abe42487fbad794c in 44ms, sequenceid=2, compaction requested=false; wal=null at 1733862502619 (+12 ms)Cleaning up temporary data from old regions at 1733862502621 (+2 ms)Region opened successfully at 1733862502632 (+11 ms) 2024-12-10T20:28:22,653 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=387 (was 377) Potentially hanging thread: PacketResponder: BP-76190982-172.17.0.3-1733862492788:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-76190982-172.17.0.3-1733862492788:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:43634 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:41660 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:60634 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:51980 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-76190982-172.17.0.3-1733862492788:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:44044 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:51864 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=272 (was 272), ProcessCount=11 (was 11), AvailableMemoryMB=5439 (was 5448) 2024-12-10T20:28:22,664 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=387, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=272, ProcessCount=11, AvailableMemoryMB=5438 2024-12-10T20:28:22,677 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:22,679 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:22,680 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T20:28:22,684 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-95111253, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/hregion-95111253, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:22,703 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-95111253/hregion-95111253.1733862502685, exclude list is [], retry=0 2024-12-10T20:28:22,707 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:22,707 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:22,707 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:22,710 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-95111253/hregion-95111253.1733862502685 2024-12-10T20:28:22,712 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:46745:46745)] 2024-12-10T20:28:22,712 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1f88531595d75b5357b721de31ab7705, NAME => 'testReplayEditsWrittenIntoWAL,,1733862502678.1f88531595d75b5357b721de31ab7705.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33493/hbase 2024-12-10T20:28:22,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741854_1030 (size=64) 2024-12-10T20:28:22,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741854_1030 (size=64) 2024-12-10T20:28:22,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741854_1030 (size=64) 2024-12-10T20:28:22,730 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733862502678.1f88531595d75b5357b721de31ab7705.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:22,733 INFO [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1f88531595d75b5357b721de31ab7705 2024-12-10T20:28:22,735 INFO [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f88531595d75b5357b721de31ab7705 columnFamilyName a 2024-12-10T20:28:22,735 DEBUG [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:22,736 INFO [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] regionserver.HStore(327): Store=1f88531595d75b5357b721de31ab7705/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:22,736 INFO [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1f88531595d75b5357b721de31ab7705 2024-12-10T20:28:22,738 INFO [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f88531595d75b5357b721de31ab7705 columnFamilyName b 2024-12-10T20:28:22,739 DEBUG [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:22,740 INFO [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] regionserver.HStore(327): Store=1f88531595d75b5357b721de31ab7705/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:22,740 INFO [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1f88531595d75b5357b721de31ab7705 2024-12-10T20:28:22,742 INFO [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f88531595d75b5357b721de31ab7705 columnFamilyName c 2024-12-10T20:28:22,742 DEBUG [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:22,743 INFO [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] regionserver.HStore(327): Store=1f88531595d75b5357b721de31ab7705/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:22,743 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1f88531595d75b5357b721de31ab7705 2024-12-10T20:28:22,744 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705 2024-12-10T20:28:22,744 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705 2024-12-10T20:28:22,746 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1f88531595d75b5357b721de31ab7705 2024-12-10T20:28:22,746 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1f88531595d75b5357b721de31ab7705 2024-12-10T20:28:22,746 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T20:28:22,748 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1f88531595d75b5357b721de31ab7705 2024-12-10T20:28:22,751 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:28:22,752 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1f88531595d75b5357b721de31ab7705; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75031416, jitterRate=0.11805522441864014}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T20:28:22,753 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1f88531595d75b5357b721de31ab7705: Writing region info on filesystem at 1733862502731Initializing all the Stores at 1733862502732 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862502732Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862502732Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862502733 (+1 ms)Cleaning up temporary data from old regions at 1733862502746 (+13 ms)Region opened successfully at 1733862502752 (+6 ms) 2024-12-10T20:28:22,753 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1f88531595d75b5357b721de31ab7705, disabling compactions & flushes 2024-12-10T20:28:22,753 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733862502678.1f88531595d75b5357b721de31ab7705. 2024-12-10T20:28:22,753 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733862502678.1f88531595d75b5357b721de31ab7705. 2024-12-10T20:28:22,753 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733862502678.1f88531595d75b5357b721de31ab7705. after waiting 0 ms 2024-12-10T20:28:22,753 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733862502678.1f88531595d75b5357b721de31ab7705. 2024-12-10T20:28:22,754 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733862502678.1f88531595d75b5357b721de31ab7705. 2024-12-10T20:28:22,754 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1f88531595d75b5357b721de31ab7705: Waiting for close lock at 1733862502753Disabling compacts and flushes for region at 1733862502753Disabling writes for close at 1733862502753Writing region close event to WAL at 1733862502754 (+1 ms)Closed at 1733862502754 2024-12-10T20:28:22,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741853_1029 (size=93) 2024-12-10T20:28:22,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741853_1029 (size=93) 2024-12-10T20:28:22,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741853_1029 (size=93) 2024-12-10T20:28:22,762 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T20:28:22,762 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-95111253:(num 1733862502685) 2024-12-10T20:28:22,762 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T20:28:22,766 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:22,780 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862502766, exclude list is [], retry=0 2024-12-10T20:28:22,784 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:22,784 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:22,785 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:22,787 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862502766 2024-12-10T20:28:22,787 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:40461:40461)] 2024-12-10T20:28:23,048 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862502766, size=0 (0bytes) 2024-12-10T20:28:23,048 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862502766 might be still open, length is 0 2024-12-10T20:28:23,048 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862502766 2024-12-10T20:28:23,049 WARN [IPC Server handler 0 on default port 33493 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862502766 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-10T20:28:23,050 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862502766 after 2ms 2024-12-10T20:28:25,885 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:43668 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:40165:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43668 dst: /127.0.0.1:40165 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40165 remote=/127.0.0.1:43668]. Total timeout mills is 60000, 57130 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:25,886 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:52016 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36025:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52016 dst: /127.0.0.1:36025 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:25,886 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:60664 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45577:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60664 dst: /127.0.0.1:45577 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:25,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741855_1032 (size=200589) 2024-12-10T20:28:25,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741855_1032 (size=200589) 2024-12-10T20:28:25,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741855_1032 (size=200589) 2024-12-10T20:28:26,428 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T20:28:26,472 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-10T20:28:27,051 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862502766 after 4003ms 2024-12-10T20:28:27,054 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862502766: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:27,056 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862502766 took 4008ms 2024-12-10T20:28:27,062 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733862502766.temp 2024-12-10T20:28:27,064 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/recovered.edits/0000000000000000001-wal.1733862502766.temp 2024-12-10T20:28:27,157 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862502766; continuing. 2024-12-10T20:28:27,157 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862502766 so closing down 2024-12-10T20:28:27,157 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T20:28:27,158 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T20:28:27,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741856_1033 (size=200597) 2024-12-10T20:28:27,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741856_1033 (size=200597) 2024-12-10T20:28:27,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741856_1033 (size=200597) 2024-12-10T20:28:27,164 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/recovered.edits/0000000000000000001-wal.1733862502766.temp (wrote 3002 edits, skipped 0 edits in 42 ms) 2024-12-10T20:28:27,166 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/recovered.edits/0000000000000000001-wal.1733862502766.temp to hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/recovered.edits/0000000000000003002 2024-12-10T20:28:27,167 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 110 ms; skipped=0; WAL=hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862502766, size=0, length=0, corrupted=false, cancelled=false 2024-12-10T20:28:27,167 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862502766, journal: Splitting hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862502766, size=0 (0bytes) at 1733862503048Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/recovered.edits/0000000000000000001-wal.1733862502766.temp at 1733862507064 (+4016 ms)Split 1024 edits, skipped 0 edits. at 1733862507118 (+54 ms)Split 2048 edits, skipped 0 edits. at 1733862507139 (+21 ms)Finishing writing output for hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862502766 so closing down at 1733862507157 (+18 ms)3 split writer threads finished at 1733862507158 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/recovered.edits/0000000000000000001-wal.1733862502766.temp (wrote 3002 edits, skipped 0 edits in 42 ms) at 1733862507164 (+6 ms)Rename recovered edits hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/recovered.edits/0000000000000000001-wal.1733862502766.temp to hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/recovered.edits/0000000000000003002 at 1733862507166 (+2 ms)Processed 3002 edits across 1 Regions in 110 ms; skipped=0; WAL=hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862502766, size=0, length=0, corrupted=false, cancelled=false at 1733862507167 (+1 ms) 2024-12-10T20:28:27,169 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862502766 to hdfs://localhost:33493/hbase/oldWALs/wal.1733862502766 2024-12-10T20:28:27,171 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/recovered.edits/0000000000000003002 2024-12-10T20:28:27,171 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T20:28:27,174 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:27,194 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862507175, exclude list is [], retry=0 2024-12-10T20:28:27,198 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:27,198 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:27,198 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:27,202 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862507175 2024-12-10T20:28:27,202 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:46745:46745)] 2024-12-10T20:28:27,203 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733862502678.1f88531595d75b5357b721de31ab7705.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:27,205 INFO [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1f88531595d75b5357b721de31ab7705 2024-12-10T20:28:27,206 INFO [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f88531595d75b5357b721de31ab7705 columnFamilyName a 2024-12-10T20:28:27,206 DEBUG [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:27,207 INFO [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] regionserver.HStore(327): Store=1f88531595d75b5357b721de31ab7705/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:27,207 INFO [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1f88531595d75b5357b721de31ab7705 2024-12-10T20:28:27,208 INFO [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f88531595d75b5357b721de31ab7705 columnFamilyName b 2024-12-10T20:28:27,208 DEBUG [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:27,209 INFO [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] regionserver.HStore(327): Store=1f88531595d75b5357b721de31ab7705/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:27,209 INFO [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1f88531595d75b5357b721de31ab7705 2024-12-10T20:28:27,210 INFO [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f88531595d75b5357b721de31ab7705 columnFamilyName c 2024-12-10T20:28:27,211 DEBUG [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:27,211 INFO [StoreOpener-1f88531595d75b5357b721de31ab7705-1 {}] regionserver.HStore(327): Store=1f88531595d75b5357b721de31ab7705/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:27,211 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1f88531595d75b5357b721de31ab7705 2024-12-10T20:28:27,212 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705 2024-12-10T20:28:27,215 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705 2024-12-10T20:28:27,216 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/recovered.edits/0000000000000003002 2024-12-10T20:28:27,219 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:27,252 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-10T20:28:27,585 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1f88531595d75b5357b721de31ab7705 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-10T20:28:27,625 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/.tmp/a/ee14d6c10d3b488d8a63bc71f6f5b0a0 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733862502799/Put/seqid=0 2024-12-10T20:28:27,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741858_1035 (size=50463) 2024-12-10T20:28:27,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741858_1035 (size=50463) 2024-12-10T20:28:27,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741858_1035 (size=50463) 2024-12-10T20:28:27,634 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/.tmp/a/ee14d6c10d3b488d8a63bc71f6f5b0a0 2024-12-10T20:28:27,642 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/.tmp/a/ee14d6c10d3b488d8a63bc71f6f5b0a0 as hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/a/ee14d6c10d3b488d8a63bc71f6f5b0a0 2024-12-10T20:28:27,649 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/a/ee14d6c10d3b488d8a63bc71f6f5b0a0, entries=754, sequenceid=754, filesize=49.3 K 2024-12-10T20:28:27,649 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 1f88531595d75b5357b721de31ab7705 in 64ms, sequenceid=754, compaction requested=false; wal=null 2024-12-10T20:28:27,671 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-10T20:28:27,672 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1f88531595d75b5357b721de31ab7705 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-10T20:28:27,680 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/.tmp/a/b81f82a841e34ae0b8f569fb8b5564b0 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733862502857/Put/seqid=0 2024-12-10T20:28:27,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741859_1036 (size=20072) 2024-12-10T20:28:27,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741859_1036 (size=20072) 2024-12-10T20:28:27,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741859_1036 (size=20072) 2024-12-10T20:28:27,689 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/.tmp/a/b81f82a841e34ae0b8f569fb8b5564b0 2024-12-10T20:28:27,713 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/.tmp/b/9f93e143a3834ce59a5029cb71a8b50c is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733862502882/Put/seqid=0 2024-12-10T20:28:27,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741860_1037 (size=35835) 2024-12-10T20:28:27,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741860_1037 (size=35835) 2024-12-10T20:28:27,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741860_1037 (size=35835) 2024-12-10T20:28:27,721 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/.tmp/b/9f93e143a3834ce59a5029cb71a8b50c 2024-12-10T20:28:27,728 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/.tmp/a/b81f82a841e34ae0b8f569fb8b5564b0 as hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/a/b81f82a841e34ae0b8f569fb8b5564b0 2024-12-10T20:28:27,735 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/a/b81f82a841e34ae0b8f569fb8b5564b0, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-10T20:28:27,736 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/.tmp/b/9f93e143a3834ce59a5029cb71a8b50c as hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/b/9f93e143a3834ce59a5029cb71a8b50c 2024-12-10T20:28:27,743 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/b/9f93e143a3834ce59a5029cb71a8b50c, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-10T20:28:27,744 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 1f88531595d75b5357b721de31ab7705 in 73ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-10T20:28:27,761 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-10T20:28:27,761 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1f88531595d75b5357b721de31ab7705 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-10T20:28:27,770 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/.tmp/b/4327511e727240c6920ba276915f0bbe is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733862502906/Put/seqid=0 2024-12-10T20:28:27,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741861_1038 (size=35082) 2024-12-10T20:28:27,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741861_1038 (size=35082) 2024-12-10T20:28:27,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741861_1038 (size=35082) 2024-12-10T20:28:27,781 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/.tmp/b/4327511e727240c6920ba276915f0bbe 2024-12-10T20:28:27,812 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/.tmp/c/6046dbe4fd6f45cc9089c71e05e270e1 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733862502952/Put/seqid=0 2024-12-10T20:28:27,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741862_1039 (size=20825) 2024-12-10T20:28:27,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741862_1039 (size=20825) 2024-12-10T20:28:27,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741862_1039 (size=20825) 2024-12-10T20:28:27,822 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/.tmp/c/6046dbe4fd6f45cc9089c71e05e270e1 2024-12-10T20:28:27,830 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/.tmp/b/4327511e727240c6920ba276915f0bbe as hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/b/4327511e727240c6920ba276915f0bbe 2024-12-10T20:28:27,838 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/b/4327511e727240c6920ba276915f0bbe, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-10T20:28:27,840 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/.tmp/c/6046dbe4fd6f45cc9089c71e05e270e1 as hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/c/6046dbe4fd6f45cc9089c71e05e270e1 2024-12-10T20:28:27,847 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/c/6046dbe4fd6f45cc9089c71e05e270e1, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-10T20:28:27,847 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 1f88531595d75b5357b721de31ab7705 in 86ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-10T20:28:27,858 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733862503014/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733862502678.1f88531595d75b5357b721de31ab7705. 2024-12-10T20:28:27,861 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/recovered.edits/0000000000000003002 2024-12-10T20:28:27,862 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-10T20:28:27,862 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1f88531595d75b5357b721de31ab7705 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-10T20:28:27,871 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/.tmp/c/822e8e4bb3ee40fc8a2c14feb3dcd62a is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733862502962/Put/seqid=0 2024-12-10T20:28:27,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741863_1040 (size=50301) 2024-12-10T20:28:27,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741863_1040 (size=50301) 2024-12-10T20:28:27,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741863_1040 (size=50301) 2024-12-10T20:28:27,885 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/.tmp/c/822e8e4bb3ee40fc8a2c14feb3dcd62a 2024-12-10T20:28:27,891 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 822e8e4bb3ee40fc8a2c14feb3dcd62a 2024-12-10T20:28:27,893 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/.tmp/c/822e8e4bb3ee40fc8a2c14feb3dcd62a as hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/c/822e8e4bb3ee40fc8a2c14feb3dcd62a 2024-12-10T20:28:27,900 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 822e8e4bb3ee40fc8a2c14feb3dcd62a 2024-12-10T20:28:27,900 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/c/822e8e4bb3ee40fc8a2c14feb3dcd62a, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-10T20:28:27,901 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 1f88531595d75b5357b721de31ab7705 in 39ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-10T20:28:27,902 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/recovered.edits/0000000000000003002 2024-12-10T20:28:27,903 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1f88531595d75b5357b721de31ab7705 2024-12-10T20:28:27,903 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1f88531595d75b5357b721de31ab7705 2024-12-10T20:28:27,904 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-10T20:28:27,906 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1f88531595d75b5357b721de31ab7705 2024-12-10T20:28:27,908 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenIntoWAL/1f88531595d75b5357b721de31ab7705/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-10T20:28:27,910 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1f88531595d75b5357b721de31ab7705; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58821086, jitterRate=-0.12349751591682434}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-10T20:28:27,910 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1f88531595d75b5357b721de31ab7705: Writing region info on filesystem at 1733862507203Initializing all the Stores at 1733862507204 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862507204Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862507204Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862507204Cleaning up temporary data from old regions at 1733862507903 (+699 ms)Region opened successfully at 1733862507910 (+7 ms) 2024-12-10T20:28:27,956 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1f88531595d75b5357b721de31ab7705, disabling compactions & flushes 2024-12-10T20:28:27,956 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733862502678.1f88531595d75b5357b721de31ab7705. 2024-12-10T20:28:27,956 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733862502678.1f88531595d75b5357b721de31ab7705. 2024-12-10T20:28:27,956 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733862502678.1f88531595d75b5357b721de31ab7705. after waiting 0 ms 2024-12-10T20:28:27,956 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733862502678.1f88531595d75b5357b721de31ab7705. 2024-12-10T20:28:27,958 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733862502678.1f88531595d75b5357b721de31ab7705. 2024-12-10T20:28:27,958 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1f88531595d75b5357b721de31ab7705: Waiting for close lock at 1733862507956Disabling compacts and flushes for region at 1733862507956Disabling writes for close at 1733862507956Writing region close event to WAL at 1733862507958 (+2 ms)Closed at 1733862507958 2024-12-10T20:28:27,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741857_1034 (size=93) 2024-12-10T20:28:27,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741857_1034 (size=93) 2024-12-10T20:28:27,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741857_1034 (size=93) 2024-12-10T20:28:27,965 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T20:28:27,965 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733862507175) 2024-12-10T20:28:27,977 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=401 (was 387) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:33493 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1210587230_22 at /127.0.0.1:60724 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:32823 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:32823 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:33493 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:43849 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1210587230_22 at /127.0.0.1:52034 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1210587230_22 at /127.0.0.1:60698 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43849 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1210587230_22 at /127.0.0.1:43684 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=315 (was 272) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5358 (was 5438) 2024-12-10T20:28:27,989 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=401, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=315, ProcessCount=11, AvailableMemoryMB=5358 2024-12-10T20:28:28,003 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:28,006 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:28,007 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T20:28:28,010 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-21203477, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/hregion-21203477, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:28,022 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-21203477/hregion-21203477.1733862508010, exclude list is [], retry=0 2024-12-10T20:28:28,026 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:28,026 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:28,028 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:28,033 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-21203477/hregion-21203477.1733862508010 2024-12-10T20:28:28,034 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:33825:33825)] 2024-12-10T20:28:28,034 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 48e53aa531b57f7cfd806ed6f2bedfd7, NAME => 'test2727,,1733862508004.48e53aa531b57f7cfd806ed6f2bedfd7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33493/hbase 2024-12-10T20:28:28,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741865_1042 (size=43) 2024-12-10T20:28:28,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741865_1042 (size=43) 2024-12-10T20:28:28,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741865_1042 (size=43) 2024-12-10T20:28:28,050 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733862508004.48e53aa531b57f7cfd806ed6f2bedfd7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:28,052 INFO [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 48e53aa531b57f7cfd806ed6f2bedfd7 2024-12-10T20:28:28,054 INFO [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48e53aa531b57f7cfd806ed6f2bedfd7 columnFamilyName a 2024-12-10T20:28:28,054 DEBUG [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:28,055 INFO [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] regionserver.HStore(327): Store=48e53aa531b57f7cfd806ed6f2bedfd7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:28,055 INFO [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 48e53aa531b57f7cfd806ed6f2bedfd7 2024-12-10T20:28:28,057 INFO [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48e53aa531b57f7cfd806ed6f2bedfd7 columnFamilyName b 2024-12-10T20:28:28,057 DEBUG [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:28,058 INFO [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] regionserver.HStore(327): Store=48e53aa531b57f7cfd806ed6f2bedfd7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:28,058 INFO [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 48e53aa531b57f7cfd806ed6f2bedfd7 2024-12-10T20:28:28,060 INFO [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48e53aa531b57f7cfd806ed6f2bedfd7 columnFamilyName c 2024-12-10T20:28:28,060 DEBUG [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:28,061 INFO [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] regionserver.HStore(327): Store=48e53aa531b57f7cfd806ed6f2bedfd7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:28,061 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 48e53aa531b57f7cfd806ed6f2bedfd7 2024-12-10T20:28:28,062 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7 2024-12-10T20:28:28,062 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7 2024-12-10T20:28:28,063 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 48e53aa531b57f7cfd806ed6f2bedfd7 2024-12-10T20:28:28,063 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 48e53aa531b57f7cfd806ed6f2bedfd7 2024-12-10T20:28:28,064 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T20:28:28,065 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 48e53aa531b57f7cfd806ed6f2bedfd7 2024-12-10T20:28:28,068 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:28:28,069 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 48e53aa531b57f7cfd806ed6f2bedfd7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72206411, jitterRate=0.07595936954021454}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T20:28:28,070 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 48e53aa531b57f7cfd806ed6f2bedfd7: Writing region info on filesystem at 1733862508050Initializing all the Stores at 1733862508051 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862508051Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862508052 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862508052Cleaning up temporary data from old regions at 1733862508063 (+11 ms)Region opened successfully at 1733862508070 (+7 ms) 2024-12-10T20:28:28,070 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 48e53aa531b57f7cfd806ed6f2bedfd7, disabling compactions & flushes 2024-12-10T20:28:28,070 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733862508004.48e53aa531b57f7cfd806ed6f2bedfd7. 2024-12-10T20:28:28,070 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733862508004.48e53aa531b57f7cfd806ed6f2bedfd7. 2024-12-10T20:28:28,070 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733862508004.48e53aa531b57f7cfd806ed6f2bedfd7. after waiting 0 ms 2024-12-10T20:28:28,070 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733862508004.48e53aa531b57f7cfd806ed6f2bedfd7. 2024-12-10T20:28:28,071 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733862508004.48e53aa531b57f7cfd806ed6f2bedfd7. 2024-12-10T20:28:28,071 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 48e53aa531b57f7cfd806ed6f2bedfd7: Waiting for close lock at 1733862508070Disabling compacts and flushes for region at 1733862508070Disabling writes for close at 1733862508070Writing region close event to WAL at 1733862508071 (+1 ms)Closed at 1733862508071 2024-12-10T20:28:28,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741864_1041 (size=93) 2024-12-10T20:28:28,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741864_1041 (size=93) 2024-12-10T20:28:28,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741864_1041 (size=93) 2024-12-10T20:28:28,077 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T20:28:28,078 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-21203477:(num 1733862508010) 2024-12-10T20:28:28,078 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T20:28:28,080 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:28,099 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508081, exclude list is [], retry=0 2024-12-10T20:28:28,102 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:28,102 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:28,102 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:28,104 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508081 2024-12-10T20:28:28,104 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:46745:46745)] 2024-12-10T20:28:28,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-10T20:28:28,124 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-10T20:28:28,126 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T20:28:28,126 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-10T20:28:28,126 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-10T20:28:28,126 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-10T20:28:28,127 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-10T20:28:28,127 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-10T20:28:28,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741866_1043 (size=200357) 2024-12-10T20:28:28,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741866_1043 (size=200357) 2024-12-10T20:28:28,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741866_1043 (size=200357) 2024-12-10T20:28:28,273 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508081, size=195.7 K (200357bytes) 2024-12-10T20:28:28,273 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508081 2024-12-10T20:28:28,273 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508081 after 0ms 2024-12-10T20:28:28,277 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508081: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:28,279 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508081 took 7ms 2024-12-10T20:28:28,285 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733862508081.temp 2024-12-10T20:28:28,287 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000000001-wal.1733862508081.temp 2024-12-10T20:28:28,331 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508081 so closing down 2024-12-10T20:28:28,332 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T20:28:28,332 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T20:28:28,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741867_1044 (size=200357) 2024-12-10T20:28:28,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741867_1044 (size=200357) 2024-12-10T20:28:28,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741867_1044 (size=200357) 2024-12-10T20:28:28,336 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000000001-wal.1733862508081.temp (wrote 3000 edits, skipped 0 edits in 19 ms) 2024-12-10T20:28:28,338 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000000001-wal.1733862508081.temp to hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000003000 2024-12-10T20:28:28,339 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 60 ms; skipped=0; WAL=hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508081, size=195.7 K, length=200357, corrupted=false, cancelled=false 2024-12-10T20:28:28,339 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508081, journal: Splitting hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508081, size=195.7 K (200357bytes) at 1733862508273Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000000001-wal.1733862508081.temp at 1733862508287 (+14 ms)Split 1024 edits, skipped 0 edits. at 1733862508298 (+11 ms)Split 2048 edits, skipped 0 edits. at 1733862508315 (+17 ms)Finishing writing output for hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508081 so closing down at 1733862508332 (+17 ms)3 split writer threads finished at 1733862508332Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000000001-wal.1733862508081.temp (wrote 3000 edits, skipped 0 edits in 19 ms) at 1733862508336 (+4 ms)Rename recovered edits hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000000001-wal.1733862508081.temp to hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000003000 at 1733862508338 (+2 ms)Processed 3000 edits across 1 Regions in 60 ms; skipped=0; WAL=hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508081, size=195.7 K, length=200357, corrupted=false, cancelled=false at 1733862508339 (+1 ms) 2024-12-10T20:28:28,341 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508081 to hdfs://localhost:33493/hbase/oldWALs/wal.1733862508081 2024-12-10T20:28:28,342 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000003000 2024-12-10T20:28:28,343 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T20:28:28,345 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:28,365 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508346, exclude list is [], retry=0 2024-12-10T20:28:28,368 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:28,369 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:28,370 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:28,373 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508346 2024-12-10T20:28:28,373 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:33825:33825)] 2024-12-10T20:28:28,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741868_1045 (size=200484) 2024-12-10T20:28:28,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741868_1045 (size=200484) 2024-12-10T20:28:28,643 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508346 not finished, retry = 0 2024-12-10T20:28:28,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741868_1045 (size=200484) 2024-12-10T20:28:28,760 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508346, size=195.8 K (200484bytes) 2024-12-10T20:28:28,760 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508346 2024-12-10T20:28:28,760 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508346 after 0ms 2024-12-10T20:28:28,763 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508346: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:28,764 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508346 took 5ms 2024-12-10T20:28:28,770 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733862508346.temp 2024-12-10T20:28:28,772 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000003001-wal.1733862508346.temp 2024-12-10T20:28:28,799 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508346 so closing down 2024-12-10T20:28:28,799 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T20:28:28,799 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T20:28:28,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741869_1046 (size=200484) 2024-12-10T20:28:28,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741869_1046 (size=200484) 2024-12-10T20:28:28,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741869_1046 (size=200484) 2024-12-10T20:28:28,806 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000003001-wal.1733862508346.temp (wrote 3000 edits, skipped 0 edits in 14 ms) 2024-12-10T20:28:28,807 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000003001-wal.1733862508346.temp to hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000006000 2024-12-10T20:28:28,808 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508346, size=195.8 K, length=200484, corrupted=false, cancelled=false 2024-12-10T20:28:28,808 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508346, journal: Splitting hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508346, size=195.8 K (200484bytes) at 1733862508760Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000003001-wal.1733862508346.temp at 1733862508772 (+12 ms)Split 1024 edits, skipped 0 edits. at 1733862508777 (+5 ms)Split 2048 edits, skipped 0 edits. at 1733862508788 (+11 ms)Finishing writing output for hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508346 so closing down at 1733862508799 (+11 ms)3 split writer threads finished at 1733862508799Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000003001-wal.1733862508346.temp (wrote 3000 edits, skipped 0 edits in 14 ms) at 1733862508806 (+7 ms)Rename recovered edits hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000003001-wal.1733862508346.temp to hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000006000 at 1733862508807 (+1 ms)Processed 3000 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508346, size=195.8 K, length=200484, corrupted=false, cancelled=false at 1733862508808 (+1 ms) 2024-12-10T20:28:28,810 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508346 to hdfs://localhost:33493/hbase/oldWALs/wal.1733862508346 2024-12-10T20:28:28,811 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000006000 2024-12-10T20:28:28,811 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T20:28:28,814 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/test2727-manual,16010,1733862508003, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:28,832 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508814, exclude list is [], retry=0 2024-12-10T20:28:28,835 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:28,836 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:28,836 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:28,840 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733862508003/wal.1733862508814 2024-12-10T20:28:28,841 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:40461:40461)] 2024-12-10T20:28:28,841 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 48e53aa531b57f7cfd806ed6f2bedfd7, NAME => 'test2727,,1733862508004.48e53aa531b57f7cfd806ed6f2bedfd7.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:28,841 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733862508004.48e53aa531b57f7cfd806ed6f2bedfd7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:28,841 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 48e53aa531b57f7cfd806ed6f2bedfd7 2024-12-10T20:28:28,841 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 48e53aa531b57f7cfd806ed6f2bedfd7 2024-12-10T20:28:28,843 INFO [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 48e53aa531b57f7cfd806ed6f2bedfd7 2024-12-10T20:28:28,844 INFO [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48e53aa531b57f7cfd806ed6f2bedfd7 columnFamilyName a 2024-12-10T20:28:28,844 DEBUG [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:28,845 INFO [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] regionserver.HStore(327): Store=48e53aa531b57f7cfd806ed6f2bedfd7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:28,845 INFO [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 48e53aa531b57f7cfd806ed6f2bedfd7 2024-12-10T20:28:28,846 INFO [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48e53aa531b57f7cfd806ed6f2bedfd7 columnFamilyName b 2024-12-10T20:28:28,847 DEBUG [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:28,847 INFO [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] regionserver.HStore(327): Store=48e53aa531b57f7cfd806ed6f2bedfd7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:28,848 INFO [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 48e53aa531b57f7cfd806ed6f2bedfd7 2024-12-10T20:28:28,849 INFO [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48e53aa531b57f7cfd806ed6f2bedfd7 columnFamilyName c 2024-12-10T20:28:28,849 DEBUG [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:28,850 INFO [StoreOpener-48e53aa531b57f7cfd806ed6f2bedfd7-1 {}] regionserver.HStore(327): Store=48e53aa531b57f7cfd806ed6f2bedfd7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:28,850 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 48e53aa531b57f7cfd806ed6f2bedfd7 2024-12-10T20:28:28,851 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7 2024-12-10T20:28:28,853 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7 2024-12-10T20:28:28,854 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000003000 2024-12-10T20:28:28,857 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:28,892 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000003000 2024-12-10T20:28:28,894 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000006000 2024-12-10T20:28:28,897 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:28,932 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000006000 2024-12-10T20:28:28,932 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 48e53aa531b57f7cfd806ed6f2bedfd7 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-10T20:28:28,950 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/.tmp/a/99c11a3b508d4cc0a72e34a16130dfaa is 41, key is test2727/a:100/1733862508377/Put/seqid=0 2024-12-10T20:28:28,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741871_1048 (size=84227) 2024-12-10T20:28:28,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741871_1048 (size=84227) 2024-12-10T20:28:28,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741871_1048 (size=84227) 2024-12-10T20:28:28,960 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/.tmp/a/99c11a3b508d4cc0a72e34a16130dfaa 2024-12-10T20:28:28,985 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/.tmp/b/d301b710c6b84f1da4bf2a53db6697f4 is 41, key is test2727/b:100/1733862508426/Put/seqid=0 2024-12-10T20:28:28,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741872_1049 (size=84609) 2024-12-10T20:28:28,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741872_1049 (size=84609) 2024-12-10T20:28:28,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741872_1049 (size=84609) 2024-12-10T20:28:28,992 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/.tmp/b/d301b710c6b84f1da4bf2a53db6697f4 2024-12-10T20:28:29,015 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/.tmp/c/a2ddfbac73be40e6b5aa0f4c1660a3ca is 41, key is test2727/c:100/1733862508600/Put/seqid=0 2024-12-10T20:28:29,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741873_1050 (size=84609) 2024-12-10T20:28:29,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741873_1050 (size=84609) 2024-12-10T20:28:29,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741873_1050 (size=84609) 2024-12-10T20:28:29,021 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/.tmp/c/a2ddfbac73be40e6b5aa0f4c1660a3ca 2024-12-10T20:28:29,028 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/.tmp/a/99c11a3b508d4cc0a72e34a16130dfaa as hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/a/99c11a3b508d4cc0a72e34a16130dfaa 2024-12-10T20:28:29,034 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/a/99c11a3b508d4cc0a72e34a16130dfaa, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-10T20:28:29,035 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/.tmp/b/d301b710c6b84f1da4bf2a53db6697f4 as hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/b/d301b710c6b84f1da4bf2a53db6697f4 2024-12-10T20:28:29,041 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/b/d301b710c6b84f1da4bf2a53db6697f4, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-10T20:28:29,042 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/.tmp/c/a2ddfbac73be40e6b5aa0f4c1660a3ca as hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/c/a2ddfbac73be40e6b5aa0f4c1660a3ca 2024-12-10T20:28:29,049 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/c/a2ddfbac73be40e6b5aa0f4c1660a3ca, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-10T20:28:29,049 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 48e53aa531b57f7cfd806ed6f2bedfd7 in 117ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-10T20:28:29,050 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000003000 2024-12-10T20:28:29,051 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/0000000000000006000 2024-12-10T20:28:29,052 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 48e53aa531b57f7cfd806ed6f2bedfd7 2024-12-10T20:28:29,052 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 48e53aa531b57f7cfd806ed6f2bedfd7 2024-12-10T20:28:29,053 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T20:28:29,054 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 48e53aa531b57f7cfd806ed6f2bedfd7 2024-12-10T20:28:29,057 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/test2727/48e53aa531b57f7cfd806ed6f2bedfd7/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-10T20:28:29,058 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 48e53aa531b57f7cfd806ed6f2bedfd7; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64012484, jitterRate=-0.046139657497406006}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T20:28:29,059 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 48e53aa531b57f7cfd806ed6f2bedfd7: Writing region info on filesystem at 1733862508841Initializing all the Stores at 1733862508842 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862508842Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862508843 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862508843Obtaining lock to block concurrent updates at 1733862508932 (+89 ms)Preparing flush snapshotting stores in 48e53aa531b57f7cfd806ed6f2bedfd7 at 1733862508932Finished memstore snapshotting test2727,,1733862508004.48e53aa531b57f7cfd806ed6f2bedfd7., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733862508932Flushing stores of test2727,,1733862508004.48e53aa531b57f7cfd806ed6f2bedfd7. at 1733862508932Flushing 48e53aa531b57f7cfd806ed6f2bedfd7/a: creating writer at 1733862508932Flushing 48e53aa531b57f7cfd806ed6f2bedfd7/a: appending metadata at 1733862508949 (+17 ms)Flushing 48e53aa531b57f7cfd806ed6f2bedfd7/a: closing flushed file at 1733862508950 (+1 ms)Flushing 48e53aa531b57f7cfd806ed6f2bedfd7/b: creating writer at 1733862508967 (+17 ms)Flushing 48e53aa531b57f7cfd806ed6f2bedfd7/b: appending metadata at 1733862508983 (+16 ms)Flushing 48e53aa531b57f7cfd806ed6f2bedfd7/b: closing flushed file at 1733862508983Flushing 48e53aa531b57f7cfd806ed6f2bedfd7/c: creating writer at 1733862508997 (+14 ms)Flushing 48e53aa531b57f7cfd806ed6f2bedfd7/c: appending metadata at 1733862509014 (+17 ms)Flushing 48e53aa531b57f7cfd806ed6f2bedfd7/c: closing flushed file at 1733862509014Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75595629: reopening flushed file at 1733862509027 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e69a47f: reopening flushed file at 1733862509034 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@719d7a04: reopening flushed file at 1733862509041 (+7 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 48e53aa531b57f7cfd806ed6f2bedfd7 in 117ms, sequenceid=6000, compaction requested=false; wal=null at 1733862509049 (+8 ms)Cleaning up temporary data from old regions at 1733862509052 (+3 ms)Region opened successfully at 1733862509059 (+7 ms) 2024-12-10T20:28:29,060 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-10T20:28:29,060 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 48e53aa531b57f7cfd806ed6f2bedfd7, disabling compactions & flushes 2024-12-10T20:28:29,061 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733862508004.48e53aa531b57f7cfd806ed6f2bedfd7. 2024-12-10T20:28:29,061 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733862508004.48e53aa531b57f7cfd806ed6f2bedfd7. 2024-12-10T20:28:29,061 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733862508004.48e53aa531b57f7cfd806ed6f2bedfd7. after waiting 0 ms 2024-12-10T20:28:29,061 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733862508004.48e53aa531b57f7cfd806ed6f2bedfd7. 2024-12-10T20:28:29,062 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733862508004.48e53aa531b57f7cfd806ed6f2bedfd7. 2024-12-10T20:28:29,062 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 48e53aa531b57f7cfd806ed6f2bedfd7: Waiting for close lock at 1733862509060Disabling compacts and flushes for region at 1733862509060Disabling writes for close at 1733862509061 (+1 ms)Writing region close event to WAL at 1733862509062 (+1 ms)Closed at 1733862509062 2024-12-10T20:28:29,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741870_1047 (size=93) 2024-12-10T20:28:29,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741870_1047 (size=93) 2024-12-10T20:28:29,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741870_1047 (size=93) 2024-12-10T20:28:29,068 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T20:28:29,068 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733862508814) 2024-12-10T20:28:29,078 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=405 (was 401) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:60808 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:43776 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:52150 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=315 (was 315), ProcessCount=11 (was 11), AvailableMemoryMB=5185 (was 5358) 2024-12-10T20:28:29,090 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=405, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=315, ProcessCount=11, AvailableMemoryMB=5184 2024-12-10T20:28:29,102 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:29,108 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733862509102, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:29,108 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733862509108 2024-12-10T20:28:29,115 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733862509102/wal.1733862509108 2024-12-10T20:28:29,116 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:40461:40461)] 2024-12-10T20:28:29,117 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 87ec4e6f26bba288d8456f678a2b7d32, NAME => 'testSequentialEditLogSeqNum,,1733862509103.87ec4e6f26bba288d8456f678a2b7d32.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:29,118 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733862509103.87ec4e6f26bba288d8456f678a2b7d32.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:29,118 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 87ec4e6f26bba288d8456f678a2b7d32 2024-12-10T20:28:29,118 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 87ec4e6f26bba288d8456f678a2b7d32 2024-12-10T20:28:29,119 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:33493/hbase/data/default/testSequentialEditLogSeqNum/87ec4e6f26bba288d8456f678a2b7d32 doesn't exist for region: 87ec4e6f26bba288d8456f678a2b7d32 on table testSequentialEditLogSeqNum 2024-12-10T20:28:29,119 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 87ec4e6f26bba288d8456f678a2b7d32 on table testSequentialEditLogSeqNum 2024-12-10T20:28:29,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741875_1052 (size=62) 2024-12-10T20:28:29,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741875_1052 (size=62) 2024-12-10T20:28:29,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741875_1052 (size=62) 2024-12-10T20:28:29,129 INFO [StoreOpener-87ec4e6f26bba288d8456f678a2b7d32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 87ec4e6f26bba288d8456f678a2b7d32 2024-12-10T20:28:29,130 INFO [StoreOpener-87ec4e6f26bba288d8456f678a2b7d32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87ec4e6f26bba288d8456f678a2b7d32 columnFamilyName a 2024-12-10T20:28:29,130 DEBUG [StoreOpener-87ec4e6f26bba288d8456f678a2b7d32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:29,131 INFO [StoreOpener-87ec4e6f26bba288d8456f678a2b7d32-1 {}] regionserver.HStore(327): Store=87ec4e6f26bba288d8456f678a2b7d32/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:29,131 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 87ec4e6f26bba288d8456f678a2b7d32 2024-12-10T20:28:29,132 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testSequentialEditLogSeqNum/87ec4e6f26bba288d8456f678a2b7d32 2024-12-10T20:28:29,132 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testSequentialEditLogSeqNum/87ec4e6f26bba288d8456f678a2b7d32 2024-12-10T20:28:29,132 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 87ec4e6f26bba288d8456f678a2b7d32 2024-12-10T20:28:29,133 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 87ec4e6f26bba288d8456f678a2b7d32 2024-12-10T20:28:29,134 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 87ec4e6f26bba288d8456f678a2b7d32 2024-12-10T20:28:29,136 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testSequentialEditLogSeqNum/87ec4e6f26bba288d8456f678a2b7d32/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:28:29,137 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 87ec4e6f26bba288d8456f678a2b7d32; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66314754, jitterRate=-0.011833161115646362}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T20:28:29,138 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 87ec4e6f26bba288d8456f678a2b7d32: Writing region info on filesystem at 1733862509118Initializing all the Stores at 1733862509129 (+11 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862509129Cleaning up temporary data from old regions at 1733862509133 (+4 ms)Region opened successfully at 1733862509137 (+4 ms) 2024-12-10T20:28:29,153 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 87ec4e6f26bba288d8456f678a2b7d32 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-10T20:28:29,172 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testSequentialEditLogSeqNum/87ec4e6f26bba288d8456f678a2b7d32/.tmp/a/b663a8a5f61343b0bb4f76e643db3209 is 81, key is testSequentialEditLogSeqNum/a:x0/1733862509138/Put/seqid=0 2024-12-10T20:28:29,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741876_1053 (size=5833) 2024-12-10T20:28:29,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741876_1053 (size=5833) 2024-12-10T20:28:29,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741876_1053 (size=5833) 2024-12-10T20:28:29,181 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testSequentialEditLogSeqNum/87ec4e6f26bba288d8456f678a2b7d32/.tmp/a/b663a8a5f61343b0bb4f76e643db3209 2024-12-10T20:28:29,190 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testSequentialEditLogSeqNum/87ec4e6f26bba288d8456f678a2b7d32/.tmp/a/b663a8a5f61343b0bb4f76e643db3209 as hdfs://localhost:33493/hbase/data/default/testSequentialEditLogSeqNum/87ec4e6f26bba288d8456f678a2b7d32/a/b663a8a5f61343b0bb4f76e643db3209 2024-12-10T20:28:29,197 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testSequentialEditLogSeqNum/87ec4e6f26bba288d8456f678a2b7d32/a/b663a8a5f61343b0bb4f76e643db3209, entries=10, sequenceid=13, filesize=5.7 K 2024-12-10T20:28:29,199 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 87ec4e6f26bba288d8456f678a2b7d32 in 46ms, sequenceid=13, compaction requested=false 2024-12-10T20:28:29,199 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 87ec4e6f26bba288d8456f678a2b7d32: 2024-12-10T20:28:29,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T20:28:29,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T20:28:29,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T20:28:29,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T20:28:29,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-10T20:28:29,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741874_1051 (size=1616) 2024-12-10T20:28:29,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741874_1051 (size=1616) 2024-12-10T20:28:29,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741874_1051 (size=1616) 2024-12-10T20:28:29,227 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33493/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733862509102/wal.1733862509108, size=1.6 K (1616bytes) 2024-12-10T20:28:29,227 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33493/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733862509102/wal.1733862509108 2024-12-10T20:28:29,228 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33493/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733862509102/wal.1733862509108 after 0ms 2024-12-10T20:28:29,232 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733862509102/wal.1733862509108: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:29,232 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33493/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733862509102/wal.1733862509108 took 5ms 2024-12-10T20:28:29,235 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33493/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733862509102/wal.1733862509108 so closing down 2024-12-10T20:28:29,235 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T20:28:29,236 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733862509108.temp 2024-12-10T20:28:29,237 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testSequentialEditLogSeqNum/87ec4e6f26bba288d8456f678a2b7d32/recovered.edits/0000000000000000003-wal.1733862509108.temp 2024-12-10T20:28:29,238 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T20:28:29,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741877_1054 (size=1175) 2024-12-10T20:28:29,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741877_1054 (size=1175) 2024-12-10T20:28:29,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741877_1054 (size=1175) 2024-12-10T20:28:29,245 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testSequentialEditLogSeqNum/87ec4e6f26bba288d8456f678a2b7d32/recovered.edits/0000000000000000003-wal.1733862509108.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-10T20:28:29,247 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33493/hbase/data/default/testSequentialEditLogSeqNum/87ec4e6f26bba288d8456f678a2b7d32/recovered.edits/0000000000000000003-wal.1733862509108.temp to hdfs://localhost:33493/hbase/data/default/testSequentialEditLogSeqNum/87ec4e6f26bba288d8456f678a2b7d32/recovered.edits/0000000000000000020 2024-12-10T20:28:29,247 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:33493/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733862509102/wal.1733862509108, size=1.6 K, length=1616, corrupted=false, cancelled=false 2024-12-10T20:28:29,247 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33493/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733862509102/wal.1733862509108, journal: Splitting hdfs://localhost:33493/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733862509102/wal.1733862509108, size=1.6 K (1616bytes) at 1733862509227Finishing writing output for hdfs://localhost:33493/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733862509102/wal.1733862509108 so closing down at 1733862509235 (+8 ms)Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testSequentialEditLogSeqNum/87ec4e6f26bba288d8456f678a2b7d32/recovered.edits/0000000000000000003-wal.1733862509108.temp at 1733862509237 (+2 ms)3 split writer threads finished at 1733862509238 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testSequentialEditLogSeqNum/87ec4e6f26bba288d8456f678a2b7d32/recovered.edits/0000000000000000003-wal.1733862509108.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733862509245 (+7 ms)Rename recovered edits hdfs://localhost:33493/hbase/data/default/testSequentialEditLogSeqNum/87ec4e6f26bba288d8456f678a2b7d32/recovered.edits/0000000000000000003-wal.1733862509108.temp to hdfs://localhost:33493/hbase/data/default/testSequentialEditLogSeqNum/87ec4e6f26bba288d8456f678a2b7d32/recovered.edits/0000000000000000020 at 1733862509247 (+2 ms)Processed 17 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:33493/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733862509102/wal.1733862509108, size=1.6 K, length=1616, corrupted=false, cancelled=false at 1733862509247 2024-12-10T20:28:29,263 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=410 (was 405) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:60808 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:52150 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=315 (was 315), ProcessCount=11 (was 11), AvailableMemoryMB=5183 (was 5184) 2024-12-10T20:28:29,278 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=410, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=315, ProcessCount=11, AvailableMemoryMB=5181 2024-12-10T20:28:29,296 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:29,299 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:29,325 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T20:28:29,328 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-19691336, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/hregion-19691336, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:29,341 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-19691336/hregion-19691336.1733862509329, exclude list is [], retry=0 2024-12-10T20:28:29,344 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:29,345 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:29,345 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:29,347 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-19691336/hregion-19691336.1733862509329 2024-12-10T20:28:29,348 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:40461:40461)] 2024-12-10T20:28:29,348 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d6cbcea13ea68ba5b5e8d07f5fef54bb, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733862509297.d6cbcea13ea68ba5b5e8d07f5fef54bb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33493/hbase 2024-12-10T20:28:29,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741879_1056 (size=70) 2024-12-10T20:28:29,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741879_1056 (size=70) 2024-12-10T20:28:29,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741879_1056 (size=70) 2024-12-10T20:28:29,363 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733862509297.d6cbcea13ea68ba5b5e8d07f5fef54bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:29,365 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:29,366 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6cbcea13ea68ba5b5e8d07f5fef54bb columnFamilyName a 2024-12-10T20:28:29,367 DEBUG [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:29,367 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] regionserver.HStore(327): Store=d6cbcea13ea68ba5b5e8d07f5fef54bb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:29,367 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:29,369 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6cbcea13ea68ba5b5e8d07f5fef54bb columnFamilyName b 2024-12-10T20:28:29,369 DEBUG [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:29,370 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] regionserver.HStore(327): Store=d6cbcea13ea68ba5b5e8d07f5fef54bb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:29,370 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:29,371 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6cbcea13ea68ba5b5e8d07f5fef54bb columnFamilyName c 2024-12-10T20:28:29,371 DEBUG [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:29,372 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] regionserver.HStore(327): Store=d6cbcea13ea68ba5b5e8d07f5fef54bb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:29,372 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:29,373 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:29,373 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:29,374 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:29,374 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:29,375 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T20:28:29,376 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:29,378 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:28:29,379 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d6cbcea13ea68ba5b5e8d07f5fef54bb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65264250, jitterRate=-0.0274868905544281}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T20:28:29,380 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d6cbcea13ea68ba5b5e8d07f5fef54bb: Writing region info on filesystem at 1733862509363Initializing all the Stores at 1733862509364 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862509364Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862509364Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862509364Cleaning up temporary data from old regions at 1733862509374 (+10 ms)Region opened successfully at 1733862509380 (+6 ms) 2024-12-10T20:28:29,380 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d6cbcea13ea68ba5b5e8d07f5fef54bb, disabling compactions & flushes 2024-12-10T20:28:29,380 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733862509297.d6cbcea13ea68ba5b5e8d07f5fef54bb. 2024-12-10T20:28:29,380 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733862509297.d6cbcea13ea68ba5b5e8d07f5fef54bb. 2024-12-10T20:28:29,380 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733862509297.d6cbcea13ea68ba5b5e8d07f5fef54bb. after waiting 0 ms 2024-12-10T20:28:29,380 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733862509297.d6cbcea13ea68ba5b5e8d07f5fef54bb. 2024-12-10T20:28:29,380 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733862509297.d6cbcea13ea68ba5b5e8d07f5fef54bb. 2024-12-10T20:28:29,381 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d6cbcea13ea68ba5b5e8d07f5fef54bb: Waiting for close lock at 1733862509380Disabling compacts and flushes for region at 1733862509380Disabling writes for close at 1733862509380Writing region close event to WAL at 1733862509380Closed at 1733862509380 2024-12-10T20:28:29,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741878_1055 (size=93) 2024-12-10T20:28:29,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741878_1055 (size=93) 2024-12-10T20:28:29,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741878_1055 (size=93) 2024-12-10T20:28:29,386 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T20:28:29,386 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-19691336:(num 1733862509329) 2024-12-10T20:28:29,387 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T20:28:29,389 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:29,400 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862509389, exclude list is [], retry=0 2024-12-10T20:28:29,403 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:29,404 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:29,404 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:29,406 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862509389 2024-12-10T20:28:29,406 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:33825:33825)] 2024-12-10T20:28:29,406 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d6cbcea13ea68ba5b5e8d07f5fef54bb, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733862509297.d6cbcea13ea68ba5b5e8d07f5fef54bb.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:29,407 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733862509297.d6cbcea13ea68ba5b5e8d07f5fef54bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:29,407 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:29,407 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:29,409 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:29,410 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6cbcea13ea68ba5b5e8d07f5fef54bb columnFamilyName a 2024-12-10T20:28:29,411 DEBUG [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:29,411 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] regionserver.HStore(327): Store=d6cbcea13ea68ba5b5e8d07f5fef54bb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:29,411 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:29,412 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6cbcea13ea68ba5b5e8d07f5fef54bb columnFamilyName b 2024-12-10T20:28:29,412 DEBUG [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:29,413 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] regionserver.HStore(327): Store=d6cbcea13ea68ba5b5e8d07f5fef54bb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:29,413 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:29,414 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6cbcea13ea68ba5b5e8d07f5fef54bb columnFamilyName c 2024-12-10T20:28:29,414 DEBUG [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:29,415 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] regionserver.HStore(327): Store=d6cbcea13ea68ba5b5e8d07f5fef54bb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:29,415 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:29,416 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:29,417 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:29,418 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:29,418 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:29,418 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T20:28:29,420 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:29,421 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d6cbcea13ea68ba5b5e8d07f5fef54bb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74265036, jitterRate=0.10663527250289917}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T20:28:29,422 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d6cbcea13ea68ba5b5e8d07f5fef54bb: Writing region info on filesystem at 1733862509407Initializing all the Stores at 1733862509408 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862509408Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862509409 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862509409Cleaning up temporary data from old regions at 1733862509418 (+9 ms)Region opened successfully at 1733862509422 (+4 ms) 2024-12-10T20:28:29,426 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733862509425/Put/seqid=0 2024-12-10T20:28:29,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741881_1058 (size=4826) 2024-12-10T20:28:29,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741881_1058 (size=4826) 2024-12-10T20:28:29,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741881_1058 (size=4826) 2024-12-10T20:28:29,436 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33493/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in d6cbcea13ea68ba5b5e8d07f5fef54bb/a 2024-12-10T20:28:29,443 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-10T20:28:29,443 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-10T20:28:29,443 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d6cbcea13ea68ba5b5e8d07f5fef54bb: 2024-12-10T20:28:29,445 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/a/506541fda573462e9a6e0f99f0a85b23_SeqId_3_ 2024-12-10T20:28:29,446 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33493/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into d6cbcea13ea68ba5b5e8d07f5fef54bb/a as hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/a/506541fda573462e9a6e0f99f0a85b23_SeqId_3_ - updating store file list. 2024-12-10T20:28:29,452 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 506541fda573462e9a6e0f99f0a85b23_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-10T20:28:29,453 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/a/506541fda573462e9a6e0f99f0a85b23_SeqId_3_ into d6cbcea13ea68ba5b5e8d07f5fef54bb/a 2024-12-10T20:28:29,453 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33493/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into d6cbcea13ea68ba5b5e8d07f5fef54bb/a (new location: hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/a/506541fda573462e9a6e0f99f0a85b23_SeqId_3_) 2024-12-10T20:28:29,488 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33493/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862509389, size=0 (0bytes) 2024-12-10T20:28:29,488 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33493/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862509389 might be still open, length is 0 2024-12-10T20:28:29,489 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33493/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862509389 2024-12-10T20:28:29,489 WARN [IPC Server handler 4 on default port 33493 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862509389 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-10T20:28:29,489 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33493/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862509389 after 0ms 2024-12-10T20:28:31,868 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:43836 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:40165:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43836 dst: /127.0.0.1:40165 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40165 remote=/127.0.0.1:43836]. Total timeout mills is 60000, 57591 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:31,869 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:60904 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45577:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60904 dst: /127.0.0.1:45577 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:31,869 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:52258 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36025:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52258 dst: /127.0.0.1:36025 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:31,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741880_1059 (size=434) 2024-12-10T20:28:31,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741880_1059 (size=434) 2024-12-10T20:28:33,490 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33493/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862509389 after 4001ms 2024-12-10T20:28:33,497 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862509389: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:33,497 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33493/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862509389 took 4009ms 2024-12-10T20:28:33,501 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33493/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862509389; continuing. 2024-12-10T20:28:33,501 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33493/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862509389 so closing down 2024-12-10T20:28:33,501 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T20:28:33,503 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733862509389.temp 2024-12-10T20:28:33,504 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/recovered.edits/0000000000000000005-wal.1733862509389.temp 2024-12-10T20:28:33,504 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T20:28:33,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741882_1060 (size=236) 2024-12-10T20:28:33,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741882_1060 (size=236) 2024-12-10T20:28:33,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741882_1060 (size=236) 2024-12-10T20:28:33,511 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/recovered.edits/0000000000000000005-wal.1733862509389.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-10T20:28:33,513 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/recovered.edits/0000000000000000005-wal.1733862509389.temp to hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/recovered.edits/0000000000000000005 2024-12-10T20:28:33,513 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 15 ms; skipped=1; WAL=hdfs://localhost:33493/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862509389, size=0, length=0, corrupted=false, cancelled=false 2024-12-10T20:28:33,513 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33493/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862509389, journal: Splitting hdfs://localhost:33493/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862509389, size=0 (0bytes) at 1733862509488Finishing writing output for hdfs://localhost:33493/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862509389 so closing down at 1733862513501 (+4013 ms)Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/recovered.edits/0000000000000000005-wal.1733862509389.temp at 1733862513504 (+3 ms)3 split writer threads finished at 1733862513504Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/recovered.edits/0000000000000000005-wal.1733862509389.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733862513511 (+7 ms)Rename recovered edits hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/recovered.edits/0000000000000000005-wal.1733862509389.temp to hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/recovered.edits/0000000000000000005 at 1733862513513 (+2 ms)Processed 2 edits across 1 Regions in 15 ms; skipped=1; WAL=hdfs://localhost:33493/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862509389, size=0, length=0, corrupted=false, cancelled=false at 1733862513513 2024-12-10T20:28:33,515 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33493/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862509389 to hdfs://localhost:33493/hbase/oldWALs/wal.1733862509389 2024-12-10T20:28:33,516 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/recovered.edits/0000000000000000005 2024-12-10T20:28:33,516 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T20:28:33,518 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:33,533 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862513518, exclude list is [], retry=0 2024-12-10T20:28:33,535 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:33,536 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:33,536 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:33,537 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862513518 2024-12-10T20:28:33,538 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:46745:46745)] 2024-12-10T20:28:33,538 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d6cbcea13ea68ba5b5e8d07f5fef54bb, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733862509297.d6cbcea13ea68ba5b5e8d07f5fef54bb.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:33,538 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733862509297.d6cbcea13ea68ba5b5e8d07f5fef54bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:33,538 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:33,538 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:33,540 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:33,541 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6cbcea13ea68ba5b5e8d07f5fef54bb columnFamilyName a 2024-12-10T20:28:33,541 DEBUG [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:33,547 DEBUG [StoreFileOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 506541fda573462e9a6e0f99f0a85b23_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-10T20:28:33,547 DEBUG [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/a/506541fda573462e9a6e0f99f0a85b23_SeqId_3_ 2024-12-10T20:28:33,547 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] regionserver.HStore(327): Store=d6cbcea13ea68ba5b5e8d07f5fef54bb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:33,548 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:33,549 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6cbcea13ea68ba5b5e8d07f5fef54bb columnFamilyName b 2024-12-10T20:28:33,549 DEBUG [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:33,549 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] regionserver.HStore(327): Store=d6cbcea13ea68ba5b5e8d07f5fef54bb/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:33,549 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:33,551 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d6cbcea13ea68ba5b5e8d07f5fef54bb columnFamilyName c 2024-12-10T20:28:33,551 DEBUG [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:33,551 INFO [StoreOpener-d6cbcea13ea68ba5b5e8d07f5fef54bb-1 {}] regionserver.HStore(327): Store=d6cbcea13ea68ba5b5e8d07f5fef54bb/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:33,551 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:33,552 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:33,554 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:33,554 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/recovered.edits/0000000000000000005 2024-12-10T20:28:33,556 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:33,557 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/recovered.edits/0000000000000000005 2024-12-10T20:28:33,557 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d6cbcea13ea68ba5b5e8d07f5fef54bb 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-10T20:28:33,570 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/.tmp/a/8f8adb3c853e4bafac6621340d1a1abb is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733862509458/Put/seqid=0 2024-12-10T20:28:33,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741884_1062 (size=5149) 2024-12-10T20:28:33,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741884_1062 (size=5149) 2024-12-10T20:28:33,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741884_1062 (size=5149) 2024-12-10T20:28:33,577 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/.tmp/a/8f8adb3c853e4bafac6621340d1a1abb 2024-12-10T20:28:33,583 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/.tmp/a/8f8adb3c853e4bafac6621340d1a1abb as hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/a/8f8adb3c853e4bafac6621340d1a1abb 2024-12-10T20:28:33,589 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/a/8f8adb3c853e4bafac6621340d1a1abb, entries=1, sequenceid=5, filesize=5.0 K 2024-12-10T20:28:33,589 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for d6cbcea13ea68ba5b5e8d07f5fef54bb in 32ms, sequenceid=5, compaction requested=false; wal=null 2024-12-10T20:28:33,590 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/recovered.edits/0000000000000000005 2024-12-10T20:28:33,591 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:33,591 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:33,592 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T20:28:33,593 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d6cbcea13ea68ba5b5e8d07f5fef54bb 2024-12-10T20:28:33,595 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d6cbcea13ea68ba5b5e8d07f5fef54bb/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-10T20:28:33,596 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d6cbcea13ea68ba5b5e8d07f5fef54bb; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65926498, jitterRate=-0.017618626356124878}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T20:28:33,596 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d6cbcea13ea68ba5b5e8d07f5fef54bb: Writing region info on filesystem at 1733862513538Initializing all the Stores at 1733862513539 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862513539Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862513540 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862513540Obtaining lock to block concurrent updates at 1733862513557 (+17 ms)Preparing flush snapshotting stores in d6cbcea13ea68ba5b5e8d07f5fef54bb at 1733862513557Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733862509297.d6cbcea13ea68ba5b5e8d07f5fef54bb., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733862513558 (+1 ms)Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733862509297.d6cbcea13ea68ba5b5e8d07f5fef54bb. at 1733862513558Flushing d6cbcea13ea68ba5b5e8d07f5fef54bb/a: creating writer at 1733862513558Flushing d6cbcea13ea68ba5b5e8d07f5fef54bb/a: appending metadata at 1733862513570 (+12 ms)Flushing d6cbcea13ea68ba5b5e8d07f5fef54bb/a: closing flushed file at 1733862513570Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76b103bd: reopening flushed file at 1733862513582 (+12 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for d6cbcea13ea68ba5b5e8d07f5fef54bb in 32ms, sequenceid=5, compaction requested=false; wal=null at 1733862513589 (+7 ms)Cleaning up temporary data from old regions at 1733862513591 (+2 ms)Region opened successfully at 1733862513596 (+5 ms) 2024-12-10T20:28:33,599 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d6cbcea13ea68ba5b5e8d07f5fef54bb, disabling compactions & flushes 2024-12-10T20:28:33,599 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733862509297.d6cbcea13ea68ba5b5e8d07f5fef54bb. 2024-12-10T20:28:33,600 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733862509297.d6cbcea13ea68ba5b5e8d07f5fef54bb. 2024-12-10T20:28:33,600 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733862509297.d6cbcea13ea68ba5b5e8d07f5fef54bb. after waiting 0 ms 2024-12-10T20:28:33,600 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733862509297.d6cbcea13ea68ba5b5e8d07f5fef54bb. 2024-12-10T20:28:33,600 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733862509297.d6cbcea13ea68ba5b5e8d07f5fef54bb. 2024-12-10T20:28:33,600 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d6cbcea13ea68ba5b5e8d07f5fef54bb: Waiting for close lock at 1733862513599Disabling compacts and flushes for region at 1733862513599Disabling writes for close at 1733862513600 (+1 ms)Writing region close event to WAL at 1733862513600Closed at 1733862513600 2024-12-10T20:28:33,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741883_1061 (size=93) 2024-12-10T20:28:33,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741883_1061 (size=93) 2024-12-10T20:28:33,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741883_1061 (size=93) 2024-12-10T20:28:33,605 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T20:28:33,605 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733862513518) 2024-12-10T20:28:33,616 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=411 (was 410) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:33493 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2024593989_22 at /127.0.0.1:32814 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2024593989_22 at /127.0.0.1:57508 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:33493 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1022 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=306 (was 315), ProcessCount=11 (was 11), AvailableMemoryMB=5167 (was 5181) 2024-12-10T20:28:33,627 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=411, OpenFileDescriptor=1022, MaxFileDescriptor=1048576, SystemLoadAverage=306, ProcessCount=11, AvailableMemoryMB=5167 2024-12-10T20:28:33,641 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:33,646 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T20:28:33,649 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 03a2ae042c95,34471,1733862497789 2024-12-10T20:28:33,651 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@45c9276a 2024-12-10T20:28:33,651 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T20:28:33,653 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T20:28:33,657 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T20:28:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-10T20:28:33,664 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T20:28:33,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-10T20:28:33,666 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:33,668 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T20:28:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T20:28:33,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741885_1063 (size=694) 2024-12-10T20:28:33,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741885_1063 (size=694) 2024-12-10T20:28:33,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741885_1063 (size=694) 2024-12-10T20:28:33,680 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6801f39a7c6a3dc7b05c32d2cd7c10e1, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b 2024-12-10T20:28:33,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741886_1064 (size=77) 2024-12-10T20:28:33,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741886_1064 (size=77) 2024-12-10T20:28:33,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741886_1064 (size=77) 2024-12-10T20:28:33,688 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:33,688 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 6801f39a7c6a3dc7b05c32d2cd7c10e1, disabling compactions & flushes 2024-12-10T20:28:33,688 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:33,688 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:33,688 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. after waiting 0 ms 2024-12-10T20:28:33,688 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:33,688 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:33,688 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6801f39a7c6a3dc7b05c32d2cd7c10e1: Waiting for close lock at 1733862513688Disabling compacts and flushes for region at 1733862513688Disabling writes for close at 1733862513688Writing region close event to WAL at 1733862513688Closed at 1733862513688 2024-12-10T20:28:33,690 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T20:28:33,695 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733862513690"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733862513690"}]},"ts":"1733862513690"} 2024-12-10T20:28:33,699 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T20:28:33,700 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T20:28:33,702 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733862513700"}]},"ts":"1733862513700"} 2024-12-10T20:28:33,706 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-10T20:28:33,706 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {03a2ae042c95=0} racks are {/default-rack=0} 2024-12-10T20:28:33,708 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T20:28:33,708 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T20:28:33,708 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-10T20:28:33,708 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T20:28:33,708 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T20:28:33,708 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-10T20:28:33,708 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T20:28:33,708 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T20:28:33,708 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-10T20:28:33,708 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T20:28:33,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, ASSIGN}] 2024-12-10T20:28:33,711 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, ASSIGN 2024-12-10T20:28:33,713 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, ASSIGN; state=OFFLINE, location=03a2ae042c95,37285,1733862498673; forceNewPlan=false, retain=false 2024-12-10T20:28:33,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T20:28:33,866 INFO [03a2ae042c95:34471 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T20:28:33,867 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6801f39a7c6a3dc7b05c32d2cd7c10e1, regionState=OPENING, regionLocation=03a2ae042c95,37285,1733862498673 2024-12-10T20:28:33,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, ASSIGN because future has completed 2024-12-10T20:28:33,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6801f39a7c6a3dc7b05c32d2cd7c10e1, server=03a2ae042c95,37285,1733862498673}] 2024-12-10T20:28:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T20:28:34,025 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T20:28:34,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51885, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T20:28:34,034 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:34,034 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6801f39a7c6a3dc7b05c32d2cd7c10e1, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:34,034 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,035 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:34,035 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,035 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,036 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,038 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6801f39a7c6a3dc7b05c32d2cd7c10e1 columnFamilyName cf1 2024-12-10T20:28:34,039 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:34,039 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(327): Store=6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:34,040 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,042 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6801f39a7c6a3dc7b05c32d2cd7c10e1 columnFamilyName cf2 2024-12-10T20:28:34,042 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:34,043 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(327): Store=6801f39a7c6a3dc7b05c32d2cd7c10e1/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:34,043 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,044 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,045 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,046 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,046 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,048 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-10T20:28:34,049 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,054 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:28:34,055 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6801f39a7c6a3dc7b05c32d2cd7c10e1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59274320, jitterRate=-0.11674380302429199}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-10T20:28:34,055 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,056 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6801f39a7c6a3dc7b05c32d2cd7c10e1: Running coprocessor pre-open hook at 1733862514035Writing region info on filesystem at 1733862514035Initializing all the Stores at 1733862514036 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862514036Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862514036Cleaning up temporary data from old regions at 1733862514046 (+10 ms)Running coprocessor post-open hooks at 1733862514055 (+9 ms)Region opened successfully at 1733862514056 (+1 ms) 2024-12-10T20:28:34,058 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., pid=6, masterSystemTime=1733862514025 2024-12-10T20:28:34,062 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:34,062 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:34,062 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6801f39a7c6a3dc7b05c32d2cd7c10e1, regionState=OPEN, openSeqNum=2, regionLocation=03a2ae042c95,37285,1733862498673 2024-12-10T20:28:34,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6801f39a7c6a3dc7b05c32d2cd7c10e1, server=03a2ae042c95,37285,1733862498673 because future has completed 2024-12-10T20:28:34,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-10T20:28:34,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6801f39a7c6a3dc7b05c32d2cd7c10e1, server=03a2ae042c95,37285,1733862498673 in 200 msec 2024-12-10T20:28:34,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-10T20:28:34,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, ASSIGN in 367 msec 2024-12-10T20:28:34,082 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T20:28:34,082 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733862514082"}]},"ts":"1733862514082"} 2024-12-10T20:28:34,085 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-10T20:28:34,086 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T20:28:34,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 429 msec 2024-12-10T20:28:34,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T20:28:34,298 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-10T20:28:34,298 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-10T20:28:34,299 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T20:28:34,305 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-10T20:28:34,306 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T20:28:34,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-10T20:28:34,322 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,37285,1733862498673, seqNum=2] 2024-12-10T20:28:34,324 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:28:34,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:28:34,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=6801f39a7c6a3dc7b05c32d2cd7c10e1, source=03a2ae042c95,37285,1733862498673, destination=03a2ae042c95,34629,1733862498540, warming up region on 03a2ae042c95,34629,1733862498540 2024-12-10T20:28:34,345 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=6801f39a7c6a3dc7b05c32d2cd7c10e1, source=03a2ae042c95,37285,1733862498673, destination=03a2ae042c95,34629,1733862498540, running balancer 2024-12-10T20:28:34,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, REOPEN/MOVE 2024-12-10T20:28:34,347 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, REOPEN/MOVE 2024-12-10T20:28:34,350 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6801f39a7c6a3dc7b05c32d2cd7c10e1, regionState=CLOSING, regionLocation=03a2ae042c95,37285,1733862498673 2024-12-10T20:28:34,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34629 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:34,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34629 {}] regionserver.HRegion(7855): Warmup {ENCODED => 6801f39a7c6a3dc7b05c32d2cd7c10e1, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:34,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34629 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:34,356 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, REOPEN/MOVE because future has completed 2024-12-10T20:28:34,360 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6801f39a7c6a3dc7b05c32d2cd7c10e1 columnFamilyName cf1 2024-12-10T20:28:34,360 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:34,361 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(327): Store=6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:34,361 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,361 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T20:28:34,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6801f39a7c6a3dc7b05c32d2cd7c10e1, server=03a2ae042c95,37285,1733862498673}] 2024-12-10T20:28:34,362 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6801f39a7c6a3dc7b05c32d2cd7c10e1 columnFamilyName cf2 2024-12-10T20:28:34,362 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:34,365 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(327): Store=6801f39a7c6a3dc7b05c32d2cd7c10e1/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:34,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34629 {}] regionserver.HRegion(1722): Closing 6801f39a7c6a3dc7b05c32d2cd7c10e1, disabling compactions & flushes 2024-12-10T20:28:34,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34629 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:34,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34629 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:34,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34629 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. after waiting 0 ms 2024-12-10T20:28:34,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34629 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:34,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34629 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:34,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34629 {}] regionserver.HRegion(1676): Region close journal for 6801f39a7c6a3dc7b05c32d2cd7c10e1: Waiting for close lock at 1733862514365Disabling compacts and flushes for region at 1733862514365Disabling writes for close at 1733862514366 (+1 ms)Writing region close event to WAL at 1733862514369 (+3 ms)Closed at 1733862514369 2024-12-10T20:28:34,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-10T20:28:34,522 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,522 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T20:28:34,523 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 6801f39a7c6a3dc7b05c32d2cd7c10e1, disabling compactions & flushes 2024-12-10T20:28:34,523 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:34,523 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:34,523 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. after waiting 0 ms 2024-12-10T20:28:34,523 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:34,524 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 6801f39a7c6a3dc7b05c32d2cd7c10e1 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-10T20:28:34,541 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/.tmp/cf1/c0cf8c3d3a1041b1a9a98004a8a10758 is 35, key is r1/cf1:q/1733862514327/Put/seqid=0 2024-12-10T20:28:34,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741887_1065 (size=4783) 2024-12-10T20:28:34,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741887_1065 (size=4783) 2024-12-10T20:28:34,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741887_1065 (size=4783) 2024-12-10T20:28:34,556 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/.tmp/cf1/c0cf8c3d3a1041b1a9a98004a8a10758 2024-12-10T20:28:34,569 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/.tmp/cf1/c0cf8c3d3a1041b1a9a98004a8a10758 as hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1/c0cf8c3d3a1041b1a9a98004a8a10758 2024-12-10T20:28:34,579 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1/c0cf8c3d3a1041b1a9a98004a8a10758, entries=1, sequenceid=5, filesize=4.7 K 2024-12-10T20:28:34,581 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 6801f39a7c6a3dc7b05c32d2cd7c10e1 in 56ms, sequenceid=5, compaction requested=false 2024-12-10T20:28:34,581 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-10T20:28:34,596 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-10T20:28:34,599 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:34,599 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 6801f39a7c6a3dc7b05c32d2cd7c10e1: Waiting for close lock at 1733862514523Running coprocessor pre-close hooks at 1733862514523Disabling compacts and flushes for region at 1733862514523Disabling writes for close at 1733862514523Obtaining lock to block concurrent updates at 1733862514524 (+1 ms)Preparing flush snapshotting stores in 6801f39a7c6a3dc7b05c32d2cd7c10e1 at 1733862514524Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733862514524Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. at 1733862514525 (+1 ms)Flushing 6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1: creating writer at 1733862514525Flushing 6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1: appending metadata at 1733862514540 (+15 ms)Flushing 6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1: closing flushed file at 1733862514541 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66f4486c: reopening flushed file at 1733862514568 (+27 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 6801f39a7c6a3dc7b05c32d2cd7c10e1 in 56ms, sequenceid=5, compaction requested=false at 1733862514581 (+13 ms)Writing region close event to WAL at 1733862514585 (+4 ms)Running coprocessor post-close hooks at 1733862514597 (+12 ms)Closed at 1733862514599 (+2 ms) 2024-12-10T20:28:34,600 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 6801f39a7c6a3dc7b05c32d2cd7c10e1 move to 03a2ae042c95,34629,1733862498540 record at close sequenceid=5 2024-12-10T20:28:34,604 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,604 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6801f39a7c6a3dc7b05c32d2cd7c10e1, regionState=CLOSED 2024-12-10T20:28:34,607 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6801f39a7c6a3dc7b05c32d2cd7c10e1, server=03a2ae042c95,37285,1733862498673 because future has completed 2024-12-10T20:28:34,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-10T20:28:34,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 6801f39a7c6a3dc7b05c32d2cd7c10e1, server=03a2ae042c95,37285,1733862498673 in 247 msec 2024-12-10T20:28:34,614 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, REOPEN/MOVE; state=CLOSED, location=03a2ae042c95,34629,1733862498540; forceNewPlan=false, retain=false 2024-12-10T20:28:34,765 INFO [03a2ae042c95:34471 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T20:28:34,765 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6801f39a7c6a3dc7b05c32d2cd7c10e1, regionState=OPENING, regionLocation=03a2ae042c95,34629,1733862498540 2024-12-10T20:28:34,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, REOPEN/MOVE because future has completed 2024-12-10T20:28:34,769 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6801f39a7c6a3dc7b05c32d2cd7c10e1, server=03a2ae042c95,34629,1733862498540}] 2024-12-10T20:28:34,928 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:34,928 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 6801f39a7c6a3dc7b05c32d2cd7c10e1, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:34,928 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,928 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:34,928 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,928 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,934 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,936 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6801f39a7c6a3dc7b05c32d2cd7c10e1 columnFamilyName cf1 2024-12-10T20:28:34,936 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:34,945 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1/c0cf8c3d3a1041b1a9a98004a8a10758 2024-12-10T20:28:34,945 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(327): Store=6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:34,945 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,947 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6801f39a7c6a3dc7b05c32d2cd7c10e1 columnFamilyName cf2 2024-12-10T20:28:34,947 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:34,947 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(327): Store=6801f39a7c6a3dc7b05c32d2cd7c10e1/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:34,948 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,949 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,950 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,951 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,951 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,952 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-10T20:28:34,955 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,956 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 6801f39a7c6a3dc7b05c32d2cd7c10e1; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64710100, jitterRate=-0.03574436902999878}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-10T20:28:34,956 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:34,957 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 6801f39a7c6a3dc7b05c32d2cd7c10e1: Running coprocessor pre-open hook at 1733862514929Writing region info on filesystem at 1733862514929Initializing all the Stores at 1733862514930 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862514930Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862514934 (+4 ms)Cleaning up temporary data from old regions at 1733862514951 (+17 ms)Running coprocessor post-open hooks at 1733862514956 (+5 ms)Region opened successfully at 1733862514957 (+1 ms) 2024-12-10T20:28:34,959 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., pid=9, masterSystemTime=1733862514922 2024-12-10T20:28:34,964 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6801f39a7c6a3dc7b05c32d2cd7c10e1, regionState=OPEN, openSeqNum=9, regionLocation=03a2ae042c95,34629,1733862498540 2024-12-10T20:28:34,965 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:34,966 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:34,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6801f39a7c6a3dc7b05c32d2cd7c10e1, server=03a2ae042c95,34629,1733862498540 because future has completed 2024-12-10T20:28:34,974 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34471 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=03a2ae042c95,34629,1733862498540, table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-10T20:28:34,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-10T20:28:34,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 6801f39a7c6a3dc7b05c32d2cd7c10e1, server=03a2ae042c95,34629,1733862498540 in 206 msec 2024-12-10T20:28:34,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, REOPEN/MOVE in 634 msec 2024-12-10T20:28:35,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T20:28:35,009 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T20:28:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:48470 deadline: 1733862575013, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=03a2ae042c95 port=34629 startCode=1733862498540. As of locationSeqNum=5. 2024-12-10T20:28:35,043 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,37285,1733862498673, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,37285,1733862498673, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=03a2ae042c95 port=34629 startCode=1733862498540. As of locationSeqNum=5. 2024-12-10T20:28:35,044 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,37285,1733862498673, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=03a2ae042c95 port=34629 startCode=1733862498540. As of locationSeqNum=5. 2024-12-10T20:28:35,044 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,37285,1733862498673, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,34629,1733862498540, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=03a2ae042c95 port=34629 startCode=1733862498540. As of locationSeqNum=5. 2024-12-10T20:28:35,166 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6801f39a7c6a3dc7b05c32d2cd7c10e1 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-10T20:28:35,189 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/.tmp/cf1/0815ac89d4b04579b380c8d31095d9d5 is 29, key is r1/cf1:/1733862515158/DeleteFamily/seqid=0 2024-12-10T20:28:35,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741888_1066 (size=4906) 2024-12-10T20:28:35,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741888_1066 (size=4906) 2024-12-10T20:28:35,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741888_1066 (size=4906) 2024-12-10T20:28:35,197 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/.tmp/cf1/0815ac89d4b04579b380c8d31095d9d5 2024-12-10T20:28:35,202 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0815ac89d4b04579b380c8d31095d9d5 2024-12-10T20:28:35,215 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/.tmp/cf2/5074885468de49deac3bd688800cd795 is 29, key is r1/cf2:/1733862515158/DeleteFamily/seqid=0 2024-12-10T20:28:35,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741889_1067 (size=4906) 2024-12-10T20:28:35,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741889_1067 (size=4906) 2024-12-10T20:28:35,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741889_1067 (size=4906) 2024-12-10T20:28:35,222 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/.tmp/cf2/5074885468de49deac3bd688800cd795 2024-12-10T20:28:35,229 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5074885468de49deac3bd688800cd795 2024-12-10T20:28:35,230 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/.tmp/cf1/0815ac89d4b04579b380c8d31095d9d5 as hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1/0815ac89d4b04579b380c8d31095d9d5 2024-12-10T20:28:35,237 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0815ac89d4b04579b380c8d31095d9d5 2024-12-10T20:28:35,237 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1/0815ac89d4b04579b380c8d31095d9d5, entries=1, sequenceid=12, filesize=4.8 K 2024-12-10T20:28:35,238 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/.tmp/cf2/5074885468de49deac3bd688800cd795 as hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf2/5074885468de49deac3bd688800cd795 2024-12-10T20:28:35,245 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5074885468de49deac3bd688800cd795 2024-12-10T20:28:35,245 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf2/5074885468de49deac3bd688800cd795, entries=1, sequenceid=12, filesize=4.8 K 2024-12-10T20:28:35,246 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 6801f39a7c6a3dc7b05c32d2cd7c10e1 in 80ms, sequenceid=12, compaction requested=false 2024-12-10T20:28:35,246 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6801f39a7c6a3dc7b05c32d2cd7c10e1: 2024-12-10T20:28:35,250 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-10T20:28:35,252 DEBUG [Time-limited test {}] regionserver.HStore(1541): 6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1 is initiating major compaction (all files) 2024-12-10T20:28:35,252 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T20:28:35,252 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:35,252 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:35,253 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1/c0cf8c3d3a1041b1a9a98004a8a10758, hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1/0815ac89d4b04579b380c8d31095d9d5] into tmpdir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/.tmp, totalSize=9.5 K 2024-12-10T20:28:35,254 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c0cf8c3d3a1041b1a9a98004a8a10758, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733862514327 2024-12-10T20:28:35,255 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 0815ac89d4b04579b380c8d31095d9d5, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-10T20:28:35,265 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 6801f39a7c6a3dc7b05c32d2cd7c10e1#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:28:35,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741890_1068 (size=4626) 2024-12-10T20:28:35,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741890_1068 (size=4626) 2024-12-10T20:28:35,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741890_1068 (size=4626) 2024-12-10T20:28:35,282 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/.tmp/cf1/731981bd59e14d3684f8231191abda8f as hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1/731981bd59e14d3684f8231191abda8f 2024-12-10T20:28:35,296 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1 of 6801f39a7c6a3dc7b05c32d2cd7c10e1 into 731981bd59e14d3684f8231191abda8f(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:28:35,296 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 6801f39a7c6a3dc7b05c32d2cd7c10e1: 2024-12-10T20:28:35,296 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-10T20:28:35,297 DEBUG [Time-limited test {}] regionserver.HStore(1541): 6801f39a7c6a3dc7b05c32d2cd7c10e1/cf2 is initiating major compaction (all files) 2024-12-10T20:28:35,297 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T20:28:35,297 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T20:28:35,297 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 6801f39a7c6a3dc7b05c32d2cd7c10e1/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:35,297 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf2/5074885468de49deac3bd688800cd795] into tmpdir=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/.tmp, totalSize=4.8 K 2024-12-10T20:28:35,298 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 5074885468de49deac3bd688800cd795, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-10T20:28:35,305 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 6801f39a7c6a3dc7b05c32d2cd7c10e1#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-10T20:28:35,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741891_1069 (size=4592) 2024-12-10T20:28:35,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741891_1069 (size=4592) 2024-12-10T20:28:35,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741891_1069 (size=4592) 2024-12-10T20:28:35,318 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/.tmp/cf2/d0f2290629a94d03a4cd44c21e9ca6da as hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf2/d0f2290629a94d03a4cd44c21e9ca6da 2024-12-10T20:28:35,325 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 6801f39a7c6a3dc7b05c32d2cd7c10e1/cf2 of 6801f39a7c6a3dc7b05c32d2cd7c10e1 into d0f2290629a94d03a4cd44c21e9ca6da(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:28:35,325 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 6801f39a7c6a3dc7b05c32d2cd7c10e1: 2024-12-10T20:28:35,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=6801f39a7c6a3dc7b05c32d2cd7c10e1, source=03a2ae042c95,34629,1733862498540, destination=03a2ae042c95,37285,1733862498673, warming up region on 03a2ae042c95,37285,1733862498673 2024-12-10T20:28:35,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=6801f39a7c6a3dc7b05c32d2cd7c10e1, source=03a2ae042c95,34629,1733862498540, destination=03a2ae042c95,37285,1733862498673, running balancer 2024-12-10T20:28:35,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, REOPEN/MOVE 2024-12-10T20:28:35,330 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, REOPEN/MOVE 2024-12-10T20:28:35,332 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6801f39a7c6a3dc7b05c32d2cd7c10e1, regionState=CLOSING, regionLocation=03a2ae042c95,34629,1733862498540 2024-12-10T20:28:35,332 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:35,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] regionserver.HRegion(7855): Warmup {ENCODED => 6801f39a7c6a3dc7b05c32d2cd7c10e1, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:35,333 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:35,334 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6801f39a7c6a3dc7b05c32d2cd7c10e1 columnFamilyName cf1 2024-12-10T20:28:35,334 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:35,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, REOPEN/MOVE because future has completed 2024-12-10T20:28:35,335 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-10T20:28:35,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6801f39a7c6a3dc7b05c32d2cd7c10e1, server=03a2ae042c95,34629,1733862498540}] 2024-12-10T20:28:35,343 INFO [StoreFileOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0815ac89d4b04579b380c8d31095d9d5 2024-12-10T20:28:35,343 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1/0815ac89d4b04579b380c8d31095d9d5 2024-12-10T20:28:35,348 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1/731981bd59e14d3684f8231191abda8f 2024-12-10T20:28:35,354 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1/c0cf8c3d3a1041b1a9a98004a8a10758 2024-12-10T20:28:35,354 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(327): Store=6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:35,354 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:35,355 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6801f39a7c6a3dc7b05c32d2cd7c10e1 columnFamilyName cf2 2024-12-10T20:28:35,355 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:35,362 INFO [StoreFileOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5074885468de49deac3bd688800cd795 2024-12-10T20:28:35,362 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf2/5074885468de49deac3bd688800cd795 2024-12-10T20:28:35,367 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf2/d0f2290629a94d03a4cd44c21e9ca6da 2024-12-10T20:28:35,367 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(327): Store=6801f39a7c6a3dc7b05c32d2cd7c10e1/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] regionserver.HRegion(1722): Closing 6801f39a7c6a3dc7b05c32d2cd7c10e1, disabling compactions & flushes 2024-12-10T20:28:35,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. after waiting 0 ms 2024-12-10T20:28:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:35,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37285 {}] regionserver.HRegion(1676): Region close journal for 6801f39a7c6a3dc7b05c32d2cd7c10e1: Waiting for close lock at 1733862515368Disabling compacts and flushes for region at 1733862515368Disabling writes for close at 1733862515368Writing region close event to WAL at 1733862515369 (+1 ms)Closed at 1733862515369 2024-12-10T20:28:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-10T20:28:35,490 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:35,490 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-10T20:28:35,490 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 6801f39a7c6a3dc7b05c32d2cd7c10e1, disabling compactions & flushes 2024-12-10T20:28:35,490 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:35,490 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:35,490 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. after waiting 0 ms 2024-12-10T20:28:35,490 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:35,491 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1/c0cf8c3d3a1041b1a9a98004a8a10758, hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1/0815ac89d4b04579b380c8d31095d9d5] to archive 2024-12-10T20:28:35,494 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:28:35,498 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1/0815ac89d4b04579b380c8d31095d9d5 to hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1/0815ac89d4b04579b380c8d31095d9d5 2024-12-10T20:28:35,499 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1/c0cf8c3d3a1041b1a9a98004a8a10758 to hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1/c0cf8c3d3a1041b1a9a98004a8a10758 2024-12-10T20:28:35,509 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf2/5074885468de49deac3bd688800cd795] to archive 2024-12-10T20:28:35,510 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:28:35,512 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf2/5074885468de49deac3bd688800cd795 to hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf2/5074885468de49deac3bd688800cd795 2024-12-10T20:28:35,516 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-10T20:28:35,517 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:35,517 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 6801f39a7c6a3dc7b05c32d2cd7c10e1: Waiting for close lock at 1733862515490Running coprocessor pre-close hooks at 1733862515490Disabling compacts and flushes for region at 1733862515490Disabling writes for close at 1733862515490Writing region close event to WAL at 1733862515513 (+23 ms)Running coprocessor post-close hooks at 1733862515517 (+4 ms)Closed at 1733862515517 2024-12-10T20:28:35,517 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 6801f39a7c6a3dc7b05c32d2cd7c10e1 move to 03a2ae042c95,37285,1733862498673 record at close sequenceid=12 2024-12-10T20:28:35,520 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:35,520 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6801f39a7c6a3dc7b05c32d2cd7c10e1, regionState=CLOSED 2024-12-10T20:28:35,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6801f39a7c6a3dc7b05c32d2cd7c10e1, server=03a2ae042c95,34629,1733862498540 because future has completed 2024-12-10T20:28:35,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-10T20:28:35,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 6801f39a7c6a3dc7b05c32d2cd7c10e1, server=03a2ae042c95,34629,1733862498540 in 189 msec 2024-12-10T20:28:35,528 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, REOPEN/MOVE; state=CLOSED, location=03a2ae042c95,37285,1733862498673; forceNewPlan=false, retain=false 2024-12-10T20:28:35,678 INFO [03a2ae042c95:34471 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T20:28:35,679 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6801f39a7c6a3dc7b05c32d2cd7c10e1, regionState=OPENING, regionLocation=03a2ae042c95,37285,1733862498673 2024-12-10T20:28:35,682 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, REOPEN/MOVE because future has completed 2024-12-10T20:28:35,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6801f39a7c6a3dc7b05c32d2cd7c10e1, server=03a2ae042c95,37285,1733862498673}] 2024-12-10T20:28:35,841 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:35,841 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 6801f39a7c6a3dc7b05c32d2cd7c10e1, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:35,842 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:35,842 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:35,842 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:35,842 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:35,844 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:35,846 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6801f39a7c6a3dc7b05c32d2cd7c10e1 columnFamilyName cf1 2024-12-10T20:28:35,846 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:35,856 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1/731981bd59e14d3684f8231191abda8f 2024-12-10T20:28:35,856 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(327): Store=6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:35,856 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:35,857 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6801f39a7c6a3dc7b05c32d2cd7c10e1 columnFamilyName cf2 2024-12-10T20:28:35,857 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:35,863 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf2/d0f2290629a94d03a4cd44c21e9ca6da 2024-12-10T20:28:35,863 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(327): Store=6801f39a7c6a3dc7b05c32d2cd7c10e1/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:35,863 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:35,864 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:35,865 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:35,866 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:35,866 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:35,867 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-10T20:28:35,868 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:35,869 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 6801f39a7c6a3dc7b05c32d2cd7c10e1; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68528192, jitterRate=0.021149635314941406}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-10T20:28:35,869 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:35,870 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 6801f39a7c6a3dc7b05c32d2cd7c10e1: Running coprocessor pre-open hook at 1733862515842Writing region info on filesystem at 1733862515842Initializing all the Stores at 1733862515844 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862515844Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862515844Cleaning up temporary data from old regions at 1733862515866 (+22 ms)Running coprocessor post-open hooks at 1733862515869 (+3 ms)Region opened successfully at 1733862515870 (+1 ms) 2024-12-10T20:28:35,871 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., pid=12, masterSystemTime=1733862515835 2024-12-10T20:28:35,874 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:35,874 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:35,874 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6801f39a7c6a3dc7b05c32d2cd7c10e1, regionState=OPEN, openSeqNum=18, regionLocation=03a2ae042c95,37285,1733862498673 2024-12-10T20:28:35,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6801f39a7c6a3dc7b05c32d2cd7c10e1, server=03a2ae042c95,37285,1733862498673 because future has completed 2024-12-10T20:28:35,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-10T20:28:35,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 6801f39a7c6a3dc7b05c32d2cd7c10e1, server=03a2ae042c95,37285,1733862498673 in 195 msec 2024-12-10T20:28:35,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, REOPEN/MOVE in 552 msec 2024-12-10T20:28:35,933 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T20:28:35,934 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T20:28:35,936 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 03a2ae042c95,37285,1733862498673: testing ***** 2024-12-10T20:28:35,936 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-10T20:28:35,938 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-10T20:28:35,938 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-10T20:28:35,940 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-10T20:28:35,941 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-10T20:28:35,947 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 490094736 }, "NonHeapMemoryUsage": { "committed": 170590208, "init": 7667712, "max": -1, "used": 167988376 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "03a2ae042c95", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 12, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 6, "ProcessCallTime_75th_percentile": 9, "ProcessCallTime_90th_percentile": 10, "ProcessCallTime_95th_percentile": 11, "ProcessCallTime_98th_percentile": 11, "ProcessCallTime_99th_percentile": 11, "ProcessCallTime_99.9th_percentile": 11, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 0, "TotalCallTime_max": 12, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 6, "TotalCallTime_75th_percentile": 9, "TotalCallTime_90th_percentile": 10, "TotalCallTime_95th_percentile": 11, "TotalCallTime_98th_percentile": 11, "TotalCallTime_99th_percentile": 11, "TotalCallTime_99.9th_percentile": 11, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "03a2ae042c95", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:53600", "tag.serverName": "03a2ae042c95,41599,1733862498634", "tag.clusterId": "534d3ae7-c7ac-4fbf-af98-6ae9e35d04c4", "tag.Context": "regionserver", "tag.Hostname": "03a2ae042c95", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733862498634, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-10T20:28:35,949 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34471 {}] master.MasterRpcServices(700): 03a2ae042c95,37285,1733862498673 reported a fatal error: ***** ABORTING region server 03a2ae042c95,37285,1733862498673: testing ***** 2024-12-10T20:28:35,952 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '03a2ae042c95,37285,1733862498673' ***** 2024-12-10T20:28:35,952 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-10T20:28:35,952 INFO [RS:2;03a2ae042c95:37285 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T20:28:35,953 INFO [RS:2;03a2ae042c95:37285 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-10T20:28:35,953 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T20:28:35,953 INFO [RS:2;03a2ae042c95:37285 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-10T20:28:35,953 INFO [RS:2;03a2ae042c95:37285 {}] regionserver.HRegionServer(3091): Received CLOSE for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:35,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34629 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:45762 deadline: 1733862575953, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=03a2ae042c95 port=37285 startCode=1733862498673. As of locationSeqNum=12. 2024-12-10T20:28:35,953 INFO [RS:2;03a2ae042c95:37285 {}] regionserver.HRegionServer(956): aborting server 03a2ae042c95,37285,1733862498673 2024-12-10T20:28:35,953 INFO [RS:2;03a2ae042c95:37285 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T20:28:35,954 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,34629,1733862498540, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,34629,1733862498540, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=03a2ae042c95 port=37285 startCode=1733862498673. As of locationSeqNum=12. 2024-12-10T20:28:35,954 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6801f39a7c6a3dc7b05c32d2cd7c10e1, disabling compactions & flushes 2024-12-10T20:28:35,954 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,34629,1733862498540, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=03a2ae042c95 port=37285 startCode=1733862498673. As of locationSeqNum=12. 2024-12-10T20:28:35,954 INFO [RS:2;03a2ae042c95:37285 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;03a2ae042c95:37285. 2024-12-10T20:28:35,954 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:35,954 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,34629,1733862498540, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,37285,1733862498673, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=03a2ae042c95 port=37285 startCode=1733862498673. As of locationSeqNum=12. 2024-12-10T20:28:35,954 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:35,954 DEBUG [RS:2;03a2ae042c95:37285 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T20:28:35,954 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. after waiting 0 ms 2024-12-10T20:28:35,954 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:35,954 DEBUG [RS:2;03a2ae042c95:37285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:28:35,955 INFO [RS:2;03a2ae042c95:37285 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-10T20:28:35,955 DEBUG [RS:2;03a2ae042c95:37285 {}] regionserver.HRegionServer(1325): Online Regions={6801f39a7c6a3dc7b05c32d2cd7c10e1=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.} 2024-12-10T20:28:35,955 DEBUG [RS:2;03a2ae042c95:37285 {}] regionserver.HRegionServer(1351): Waiting on 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:35,956 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:35,956 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6801f39a7c6a3dc7b05c32d2cd7c10e1: Waiting for close lock at 1733862515953Running coprocessor pre-close hooks at 1733862515954 (+1 ms)Disabling compacts and flushes for region at 1733862515954Disabling writes for close at 1733862515954Writing region close event to WAL at 1733862515956 (+2 ms)Running coprocessor post-close hooks at 1733862515956Closed at 1733862515956 2024-12-10T20:28:35,956 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:36,043 INFO [regionserver/03a2ae042c95:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T20:28:36,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 03a2ae042c95,37285,1733862498673 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:36,070 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,37285,1733862498673, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,37285,1733862498673, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 03a2ae042c95,37285,1733862498673 aborting 2024-12-10T20:28:36,071 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,37285,1733862498673, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 03a2ae042c95,37285,1733862498673 aborting 2024-12-10T20:28:36,071 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,37285,1733862498673, seqNum=12 from cache 2024-12-10T20:28:36,156 INFO [RS:2;03a2ae042c95:37285 {}] regionserver.HRegionServer(976): stopping server 03a2ae042c95,37285,1733862498673; all regions closed. 2024-12-10T20:28:36,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741833_1009 (size=1677) 2024-12-10T20:28:36,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741833_1009 (size=1677) 2024-12-10T20:28:36,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741833_1009 (size=1677) 2024-12-10T20:28:36,165 DEBUG [RS:2;03a2ae042c95:37285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:28:36,165 INFO [RS:2;03a2ae042c95:37285 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T20:28:36,166 INFO [RS:2;03a2ae042c95:37285 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T20:28:36,166 INFO [RS:2;03a2ae042c95:37285 {}] hbase.ChoreService(370): Chore service for: regionserver/03a2ae042c95:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T20:28:36,166 INFO [regionserver/03a2ae042c95:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T20:28:36,166 INFO [RS:2;03a2ae042c95:37285 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T20:28:36,166 INFO [RS:2;03a2ae042c95:37285 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T20:28:36,166 INFO [RS:2;03a2ae042c95:37285 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T20:28:36,166 INFO [RS:2;03a2ae042c95:37285 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T20:28:36,167 INFO [RS:2;03a2ae042c95:37285 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37285 2024-12-10T20:28:36,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T20:28:36,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37285-0x100116a21c40003, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/03a2ae042c95,37285,1733862498673 2024-12-10T20:28:36,194 INFO [RS:2;03a2ae042c95:37285 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T20:28:36,196 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [03a2ae042c95,37285,1733862498673] 2024-12-10T20:28:36,216 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/03a2ae042c95,37285,1733862498673 already deleted, retry=false 2024-12-10T20:28:36,216 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 03a2ae042c95,37285,1733862498673 on 03a2ae042c95,34471,1733862497789 2024-12-10T20:28:36,220 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 03a2ae042c95,37285,1733862498673, splitWal=true, meta=false 2024-12-10T20:28:36,222 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 03a2ae042c95,37285,1733862498673 (carryingMeta=false) 03a2ae042c95,37285,1733862498673/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@295842ce[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-10T20:28:36,223 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 03a2ae042c95,37285,1733862498673, splitWal=true, meta=false 2024-12-10T20:28:36,224 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 03a2ae042c95,37285,1733862498673 had 1 regions 2024-12-10T20:28:36,225 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 03a2ae042c95,37285,1733862498673, splitWal=true, meta=false, isMeta: false 2024-12-10T20:28:36,227 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,37285,1733862498673-splitting 2024-12-10T20:28:36,228 INFO [PEWorker-2 {}] master.SplitWALManager(105): 03a2ae042c95,37285,1733862498673 WAL count=1, meta=false 2024-12-10T20:28:36,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 03a2ae042c95%2C37285%2C1733862498673.1733862500392}] 2024-12-10T20:28:36,234 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=03a2ae042c95,34629,1733862498540 2024-12-10T20:28:36,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 03a2ae042c95%2C37285%2C1733862498673.1733862500392, worker=03a2ae042c95,34629,1733862498540}] 2024-12-10T20:28:36,280 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,37285,1733862498673, seqNum=18] 2024-12-10T20:28:36,282 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 03a2ae042c95:37285 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 03a2ae042c95/172.17.0.3:37285 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:36,282 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,37285,1733862498673, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,37285,1733862498673, seqNum=18, error=java.net.ConnectException: Call to address=03a2ae042c95:37285 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 03a2ae042c95/172.17.0.3:37285 2024-12-10T20:28:36,282 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,37285,1733862498673, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-10T20:28:36,283 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,37285,1733862498673, seqNum=18 from cache 2024-12-10T20:28:36,283 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 03a2ae042c95:37285 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 03a2ae042c95/172.17.0.3:37285 2024-12-10T20:28:36,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37285-0x100116a21c40003, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T20:28:36,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37285-0x100116a21c40003, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T20:28:36,307 INFO [RS:2;03a2ae042c95:37285 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T20:28:36,307 INFO [RS:2;03a2ae042c95:37285 {}] regionserver.HRegionServer(1031): Exiting; stopping=03a2ae042c95,37285,1733862498673; zookeeper connection closed. 2024-12-10T20:28:36,307 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@277f0606 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@277f0606 2024-12-10T20:28:36,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34629 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-10T20:28:36,414 INFO [RS_LOG_REPLAY_OPS-regionserver/03a2ae042c95:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,37285,1733862498673-splitting/03a2ae042c95%2C37285%2C1733862498673.1733862500392, size=1.6 K (1677bytes) 2024-12-10T20:28:36,414 INFO [RS_LOG_REPLAY_OPS-regionserver/03a2ae042c95:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,37285,1733862498673-splitting/03a2ae042c95%2C37285%2C1733862498673.1733862500392 2024-12-10T20:28:36,415 INFO [RS_LOG_REPLAY_OPS-regionserver/03a2ae042c95:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,37285,1733862498673-splitting/03a2ae042c95%2C37285%2C1733862498673.1733862500392 after 1ms 2024-12-10T20:28:36,418 DEBUG [RS_LOG_REPLAY_OPS-regionserver/03a2ae042c95:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,37285,1733862498673-splitting/03a2ae042c95%2C37285%2C1733862498673.1733862500392: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:36,418 INFO [RS_LOG_REPLAY_OPS-regionserver/03a2ae042c95:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,37285,1733862498673-splitting/03a2ae042c95%2C37285%2C1733862498673.1733862500392 took 4ms 2024-12-10T20:28:36,426 DEBUG [RS_LOG_REPLAY_OPS-regionserver/03a2ae042c95:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 6801f39a7c6a3dc7b05c32d2cd7c10e1: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-10T20:28:36,427 DEBUG [RS_LOG_REPLAY_OPS-regionserver/03a2ae042c95:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,37285,1733862498673-splitting/03a2ae042c95%2C37285%2C1733862498673.1733862500392 so closing down 2024-12-10T20:28:36,427 DEBUG [RS_LOG_REPLAY_OPS-regionserver/03a2ae042c95:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T20:28:36,427 INFO [RS_LOG_REPLAY_OPS-regionserver/03a2ae042c95:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T20:28:36,427 INFO [RS_LOG_REPLAY_OPS-regionserver/03a2ae042c95:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 8 ms; skipped=6; WAL=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,37285,1733862498673-splitting/03a2ae042c95%2C37285%2C1733862498673.1733862500392, size=1.6 K, length=1677, corrupted=false, cancelled=false 2024-12-10T20:28:36,427 DEBUG [RS_LOG_REPLAY_OPS-regionserver/03a2ae042c95:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,37285,1733862498673-splitting/03a2ae042c95%2C37285%2C1733862498673.1733862500392, journal: Splitting hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,37285,1733862498673-splitting/03a2ae042c95%2C37285%2C1733862498673.1733862500392, size=1.6 K (1677bytes) at 1733862516414Finishing writing output for hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,37285,1733862498673-splitting/03a2ae042c95%2C37285%2C1733862498673.1733862500392 so closing down at 1733862516427 (+13 ms)3 split writer threads finished at 1733862516427Processed 6 edits across 0 Regions in 8 ms; skipped=6; WAL=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,37285,1733862498673-splitting/03a2ae042c95%2C37285%2C1733862498673.1733862500392, size=1.6 K, length=1677, corrupted=false, cancelled=false at 1733862516427 2024-12-10T20:28:36,427 DEBUG [RS_LOG_REPLAY_OPS-regionserver/03a2ae042c95:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,37285,1733862498673-splitting/03a2ae042c95%2C37285%2C1733862498673.1733862500392 2024-12-10T20:28:36,429 DEBUG [RS_LOG_REPLAY_OPS-regionserver/03a2ae042c95:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-10T20:28:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34471 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-10T20:28:36,435 INFO [PEWorker-3 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,37285,1733862498673-splitting/03a2ae042c95%2C37285%2C1733862498673.1733862500392 to hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/oldWALs 2024-12-10T20:28:36,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-10T20:28:36,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 03a2ae042c95%2C37285%2C1733862498673.1733862500392, worker=03a2ae042c95,34629,1733862498540 in 199 msec 2024-12-10T20:28:36,439 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=03a2ae042c95,34629,1733862498540 2024-12-10T20:28:36,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-10T20:28:36,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 03a2ae042c95%2C37285%2C1733862498673.1733862500392, worker=03a2ae042c95,34629,1733862498540 in 210 msec 2024-12-10T20:28:36,444 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,37285,1733862498673-splitting dir is empty, no logs to split. 2024-12-10T20:28:36,444 INFO [PEWorker-2 {}] master.SplitWALManager(105): 03a2ae042c95,37285,1733862498673 WAL count=0, meta=false 2024-12-10T20:28:36,444 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 03a2ae042c95,37285,1733862498673 WAL splitting is done? wals=0, meta=false 2024-12-10T20:28:36,446 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 03a2ae042c95,37285,1733862498673 failed, ignore...File hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/WALs/03a2ae042c95,37285,1733862498673-splitting does not exist. 2024-12-10T20:28:36,448 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, ASSIGN}] 2024-12-10T20:28:36,450 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, ASSIGN 2024-12-10T20:28:36,451 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-10T20:28:36,590 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,37285,1733862498673, seqNum=18] 2024-12-10T20:28:36,591 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 03a2ae042c95:37285 this server is in the failed servers list 2024-12-10T20:28:36,592 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,37285,1733862498673, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,37285,1733862498673, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=03a2ae042c95:37285 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 03a2ae042c95:37285 2024-12-10T20:28:36,592 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,37285,1733862498673, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 03a2ae042c95:37285 2024-12-10T20:28:36,592 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,37285,1733862498673, seqNum=18 from cache 2024-12-10T20:28:36,601 DEBUG [03a2ae042c95:34471 {}] balancer.BalancerClusterState(204): Hosts are {03a2ae042c95=0} racks are {/default-rack=0} 2024-12-10T20:28:36,601 DEBUG [03a2ae042c95:34471 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T20:28:36,601 DEBUG [03a2ae042c95:34471 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T20:28:36,601 DEBUG [03a2ae042c95:34471 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T20:28:36,601 DEBUG [03a2ae042c95:34471 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T20:28:36,601 INFO [03a2ae042c95:34471 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T20:28:36,601 INFO [03a2ae042c95:34471 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T20:28:36,601 DEBUG [03a2ae042c95:34471 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T20:28:36,602 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=6801f39a7c6a3dc7b05c32d2cd7c10e1, regionState=OPENING, regionLocation=03a2ae042c95,41599,1733862498634 2024-12-10T20:28:36,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, ASSIGN because future has completed 2024-12-10T20:28:36,605 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6801f39a7c6a3dc7b05c32d2cd7c10e1, server=03a2ae042c95,41599,1733862498634}] 2024-12-10T20:28:36,758 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T20:28:36,762 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55629, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T20:28:36,770 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:36,770 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 6801f39a7c6a3dc7b05c32d2cd7c10e1, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:36,770 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:36,771 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:36,771 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:36,771 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:36,772 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:36,773 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6801f39a7c6a3dc7b05c32d2cd7c10e1 columnFamilyName cf1 2024-12-10T20:28:36,773 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:36,779 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1/731981bd59e14d3684f8231191abda8f 2024-12-10T20:28:36,779 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(327): Store=6801f39a7c6a3dc7b05c32d2cd7c10e1/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:36,780 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:36,780 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6801f39a7c6a3dc7b05c32d2cd7c10e1 columnFamilyName cf2 2024-12-10T20:28:36,781 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:36,786 DEBUG [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/cf2/d0f2290629a94d03a4cd44c21e9ca6da 2024-12-10T20:28:36,787 INFO [StoreOpener-6801f39a7c6a3dc7b05c32d2cd7c10e1-1 {}] regionserver.HStore(327): Store=6801f39a7c6a3dc7b05c32d2cd7c10e1/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:36,787 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:36,788 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:36,789 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:36,790 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:36,790 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:36,791 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-10T20:28:36,792 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:36,793 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 6801f39a7c6a3dc7b05c32d2cd7c10e1; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62544804, jitterRate=-0.06800979375839233}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-10T20:28:36,794 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:36,794 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 6801f39a7c6a3dc7b05c32d2cd7c10e1: Running coprocessor pre-open hook at 1733862516771Writing region info on filesystem at 1733862516771Initializing all the Stores at 1733862516772 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862516772Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862516772Cleaning up temporary data from old regions at 1733862516790 (+18 ms)Running coprocessor post-open hooks at 1733862516794 (+4 ms)Region opened successfully at 1733862516794 2024-12-10T20:28:36,795 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., pid=17, masterSystemTime=1733862516757 2024-12-10T20:28:36,797 DEBUG [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:36,798 INFO [RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:36,798 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=6801f39a7c6a3dc7b05c32d2cd7c10e1, regionState=OPEN, openSeqNum=18, regionLocation=03a2ae042c95,41599,1733862498634 2024-12-10T20:28:36,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6801f39a7c6a3dc7b05c32d2cd7c10e1, server=03a2ae042c95,41599,1733862498634 because future has completed 2024-12-10T20:28:36,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-10T20:28:36,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 6801f39a7c6a3dc7b05c32d2cd7c10e1, server=03a2ae042c95,41599,1733862498634 in 197 msec 2024-12-10T20:28:36,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-10T20:28:36,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=6801f39a7c6a3dc7b05c32d2cd7c10e1, ASSIGN in 355 msec 2024-12-10T20:28:36,810 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(291): removed crashed server 03a2ae042c95,37285,1733862498673 after splitting done 2024-12-10T20:28:36,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 03a2ae042c95,37285,1733862498673, splitWal=true, meta=false in 593 msec 2024-12-10T20:28:37,114 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1., hostname=03a2ae042c95,41599,1733862498634, seqNum=18] 2024-12-10T20:28:37,114 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T20:28:37,116 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T20:28:37,130 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=414 (was 411) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1519072803_22 at /127.0.0.1:46466 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1908601102_22 at /127.0.0.1:32814 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/03a2ae042c95:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_LOG_REPLAY_OPS-regionserver/03a2ae042c95:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1908601102_22 at /127.0.0.1:57508 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1051 (was 1022) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=306 (was 306), ProcessCount=11 (was 11), AvailableMemoryMB=5128 (was 5167) 2024-12-10T20:28:37,132 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-12-10T20:28:37,143 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=414, OpenFileDescriptor=1051, MaxFileDescriptor=1048576, SystemLoadAverage=306, ProcessCount=11, AvailableMemoryMB=5127 2024-12-10T20:28:37,143 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-12-10T20:28:37,156 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:37,158 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:37,158 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T20:28:37,161 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-04594327, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/hregion-04594327, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:37,171 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-04594327/hregion-04594327.1733862517161, exclude list is [], retry=0 2024-12-10T20:28:37,174 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:37,174 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:37,174 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:37,177 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-04594327/hregion-04594327.1733862517161 2024-12-10T20:28:37,177 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:40461:40461)] 2024-12-10T20:28:37,177 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 48730ca5251b871445d39d47ab457041, NAME => 'testReplayEditsWrittenViaHRegion,,1733862517156.48730ca5251b871445d39d47ab457041.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33493/hbase 2024-12-10T20:28:37,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741893_1071 (size=67) 2024-12-10T20:28:37,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741893_1071 (size=67) 2024-12-10T20:28:37,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741893_1071 (size=67) 2024-12-10T20:28:37,188 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733862517156.48730ca5251b871445d39d47ab457041.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:37,189 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,190 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48730ca5251b871445d39d47ab457041 columnFamilyName a 2024-12-10T20:28:37,190 DEBUG [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:37,191 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] regionserver.HStore(327): Store=48730ca5251b871445d39d47ab457041/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:37,191 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,192 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48730ca5251b871445d39d47ab457041 columnFamilyName b 2024-12-10T20:28:37,192 DEBUG [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:37,193 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] regionserver.HStore(327): Store=48730ca5251b871445d39d47ab457041/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:37,193 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,194 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48730ca5251b871445d39d47ab457041 columnFamilyName c 2024-12-10T20:28:37,194 DEBUG [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:37,194 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] regionserver.HStore(327): Store=48730ca5251b871445d39d47ab457041/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:37,194 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,195 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,195 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,196 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,196 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,197 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T20:28:37,198 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,200 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:28:37,200 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 48730ca5251b871445d39d47ab457041; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68780436, jitterRate=0.024908363819122314}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T20:28:37,200 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 48730ca5251b871445d39d47ab457041: Writing region info on filesystem at 1733862517188Initializing all the Stores at 1733862517189 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862517189Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862517189Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862517189Cleaning up temporary data from old regions at 1733862517196 (+7 ms)Region opened successfully at 1733862517200 (+4 ms) 2024-12-10T20:28:37,201 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 48730ca5251b871445d39d47ab457041, disabling compactions & flushes 2024-12-10T20:28:37,201 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733862517156.48730ca5251b871445d39d47ab457041. 2024-12-10T20:28:37,201 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733862517156.48730ca5251b871445d39d47ab457041. 2024-12-10T20:28:37,201 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733862517156.48730ca5251b871445d39d47ab457041. after waiting 0 ms 2024-12-10T20:28:37,201 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733862517156.48730ca5251b871445d39d47ab457041. 2024-12-10T20:28:37,201 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733862517156.48730ca5251b871445d39d47ab457041. 2024-12-10T20:28:37,201 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 48730ca5251b871445d39d47ab457041: Waiting for close lock at 1733862517201Disabling compacts and flushes for region at 1733862517201Disabling writes for close at 1733862517201Writing region close event to WAL at 1733862517201Closed at 1733862517201 2024-12-10T20:28:37,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741892_1070 (size=93) 2024-12-10T20:28:37,203 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-04594327/hregion-04594327.1733862517161 not finished, retry = 0 2024-12-10T20:28:37,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741892_1070 (size=93) 2024-12-10T20:28:37,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741892_1070 (size=93) 2024-12-10T20:28:37,307 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T20:28:37,307 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-04594327:(num 1733862517161) 2024-12-10T20:28:37,308 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T20:28:37,310 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:37,325 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155/wal.1733862517311, exclude list is [], retry=0 2024-12-10T20:28:37,327 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:37,328 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:37,328 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:37,329 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155/wal.1733862517311 2024-12-10T20:28:37,330 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:33825:33825)] 2024-12-10T20:28:37,330 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 48730ca5251b871445d39d47ab457041, NAME => 'testReplayEditsWrittenViaHRegion,,1733862517156.48730ca5251b871445d39d47ab457041.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:37,330 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733862517156.48730ca5251b871445d39d47ab457041.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:37,330 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,330 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,331 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,332 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48730ca5251b871445d39d47ab457041 columnFamilyName a 2024-12-10T20:28:37,332 DEBUG [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:37,333 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] regionserver.HStore(327): Store=48730ca5251b871445d39d47ab457041/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:37,333 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,333 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48730ca5251b871445d39d47ab457041 columnFamilyName b 2024-12-10T20:28:37,334 DEBUG [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:37,334 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] regionserver.HStore(327): Store=48730ca5251b871445d39d47ab457041/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:37,334 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,335 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48730ca5251b871445d39d47ab457041 columnFamilyName c 2024-12-10T20:28:37,335 DEBUG [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:37,335 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] regionserver.HStore(327): Store=48730ca5251b871445d39d47ab457041/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:37,335 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,336 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,337 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,338 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,338 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,339 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T20:28:37,340 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,341 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 48730ca5251b871445d39d47ab457041; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61936001, jitterRate=-0.07708166539669037}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T20:28:37,341 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 48730ca5251b871445d39d47ab457041: Writing region info on filesystem at 1733862517330Initializing all the Stores at 1733862517331 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862517331Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862517331Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862517331Cleaning up temporary data from old regions at 1733862517338 (+7 ms)Region opened successfully at 1733862517341 (+3 ms) 2024-12-10T20:28:37,366 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 48730ca5251b871445d39d47ab457041 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-10T20:28:37,381 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/.tmp/a/f226f3b6f045455c805696cc1dc74cb5 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733862517341/Put/seqid=0 2024-12-10T20:28:37,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741895_1073 (size=5958) 2024-12-10T20:28:37,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741895_1073 (size=5958) 2024-12-10T20:28:37,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741895_1073 (size=5958) 2024-12-10T20:28:37,390 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/.tmp/a/f226f3b6f045455c805696cc1dc74cb5 2024-12-10T20:28:37,410 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/.tmp/b/42478077c40a4ba593212cedcbf27e0e is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733862517349/Put/seqid=0 2024-12-10T20:28:37,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741896_1074 (size=5958) 2024-12-10T20:28:37,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741896_1074 (size=5958) 2024-12-10T20:28:37,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741896_1074 (size=5958) 2024-12-10T20:28:37,418 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/.tmp/b/42478077c40a4ba593212cedcbf27e0e 2024-12-10T20:28:37,437 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/.tmp/c/bdbb841cca034bf78997757d368834c8 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733862517355/Put/seqid=0 2024-12-10T20:28:37,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741897_1075 (size=5958) 2024-12-10T20:28:37,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741897_1075 (size=5958) 2024-12-10T20:28:37,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741897_1075 (size=5958) 2024-12-10T20:28:37,445 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/.tmp/c/bdbb841cca034bf78997757d368834c8 2024-12-10T20:28:37,451 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/.tmp/a/f226f3b6f045455c805696cc1dc74cb5 as hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/a/f226f3b6f045455c805696cc1dc74cb5 2024-12-10T20:28:37,457 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/a/f226f3b6f045455c805696cc1dc74cb5, entries=10, sequenceid=33, filesize=5.8 K 2024-12-10T20:28:37,458 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/.tmp/b/42478077c40a4ba593212cedcbf27e0e as hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/b/42478077c40a4ba593212cedcbf27e0e 2024-12-10T20:28:37,464 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/b/42478077c40a4ba593212cedcbf27e0e, entries=10, sequenceid=33, filesize=5.8 K 2024-12-10T20:28:37,465 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/.tmp/c/bdbb841cca034bf78997757d368834c8 as hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/c/bdbb841cca034bf78997757d368834c8 2024-12-10T20:28:37,470 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/c/bdbb841cca034bf78997757d368834c8, entries=10, sequenceid=33, filesize=5.8 K 2024-12-10T20:28:37,471 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 48730ca5251b871445d39d47ab457041 in 105ms, sequenceid=33, compaction requested=false 2024-12-10T20:28:37,471 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 48730ca5251b871445d39d47ab457041: 2024-12-10T20:28:37,471 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 48730ca5251b871445d39d47ab457041, disabling compactions & flushes 2024-12-10T20:28:37,471 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733862517156.48730ca5251b871445d39d47ab457041. 2024-12-10T20:28:37,472 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733862517156.48730ca5251b871445d39d47ab457041. 2024-12-10T20:28:37,472 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733862517156.48730ca5251b871445d39d47ab457041. after waiting 0 ms 2024-12-10T20:28:37,472 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733862517156.48730ca5251b871445d39d47ab457041. 2024-12-10T20:28:37,473 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733862517156.48730ca5251b871445d39d47ab457041. 2024-12-10T20:28:37,473 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 48730ca5251b871445d39d47ab457041: Waiting for close lock at 1733862517471Disabling compacts and flushes for region at 1733862517471Disabling writes for close at 1733862517472 (+1 ms)Writing region close event to WAL at 1733862517473 (+1 ms)Closed at 1733862517473 2024-12-10T20:28:37,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741894_1072 (size=2873) 2024-12-10T20:28:37,476 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155/wal.1733862517311 not finished, retry = 0 2024-12-10T20:28:37,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741894_1072 (size=2873) 2024-12-10T20:28:37,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741894_1072 (size=2873) 2024-12-10T20:28:37,582 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/b/42478077c40a4ba593212cedcbf27e0e to hdfs://localhost:33493/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/b/42478077c40a4ba593212cedcbf27e0e 2024-12-10T20:28:37,603 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33493/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155/wal.1733862517311, size=2.8 K (2873bytes) 2024-12-10T20:28:37,603 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33493/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155/wal.1733862517311 2024-12-10T20:28:37,603 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33493/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155/wal.1733862517311 after 0ms 2024-12-10T20:28:37,605 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155/wal.1733862517311: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:37,605 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33493/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155/wal.1733862517311 took 3ms 2024-12-10T20:28:37,608 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33493/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155/wal.1733862517311 so closing down 2024-12-10T20:28:37,608 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T20:28:37,608 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733862517311.temp 2024-12-10T20:28:37,610 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/recovered.edits/0000000000000000003-wal.1733862517311.temp 2024-12-10T20:28:37,610 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T20:28:37,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741898_1076 (size=2312) 2024-12-10T20:28:37,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741898_1076 (size=2312) 2024-12-10T20:28:37,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741898_1076 (size=2312) 2024-12-10T20:28:37,618 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/recovered.edits/0000000000000000003-wal.1733862517311.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-10T20:28:37,620 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/recovered.edits/0000000000000000003-wal.1733862517311.temp to hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/recovered.edits/0000000000000000032 2024-12-10T20:28:37,620 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:33493/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155/wal.1733862517311, size=2.8 K, length=2873, corrupted=false, cancelled=false 2024-12-10T20:28:37,620 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33493/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155/wal.1733862517311, journal: Splitting hdfs://localhost:33493/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155/wal.1733862517311, size=2.8 K (2873bytes) at 1733862517603Finishing writing output for hdfs://localhost:33493/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155/wal.1733862517311 so closing down at 1733862517608 (+5 ms)Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/recovered.edits/0000000000000000003-wal.1733862517311.temp at 1733862517610 (+2 ms)3 split writer threads finished at 1733862517610Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/recovered.edits/0000000000000000003-wal.1733862517311.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733862517618 (+8 ms)Rename recovered edits hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/recovered.edits/0000000000000000003-wal.1733862517311.temp to hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/recovered.edits/0000000000000000032 at 1733862517620 (+2 ms)Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:33493/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155/wal.1733862517311, size=2.8 K, length=2873, corrupted=false, cancelled=false at 1733862517620 2024-12-10T20:28:37,622 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33493/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155/wal.1733862517311 to hdfs://localhost:33493/hbase/oldWALs/wal.1733862517311 2024-12-10T20:28:37,623 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/recovered.edits/0000000000000000032 2024-12-10T20:28:37,623 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T20:28:37,624 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:37,636 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155/wal.1733862517625, exclude list is [], retry=0 2024-12-10T20:28:37,638 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:37,639 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:37,639 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:37,640 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155/wal.1733862517625 2024-12-10T20:28:37,641 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:33825:33825)] 2024-12-10T20:28:37,641 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 48730ca5251b871445d39d47ab457041, NAME => 'testReplayEditsWrittenViaHRegion,,1733862517156.48730ca5251b871445d39d47ab457041.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:37,641 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733862517156.48730ca5251b871445d39d47ab457041.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:37,641 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,641 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,643 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,644 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48730ca5251b871445d39d47ab457041 columnFamilyName a 2024-12-10T20:28:37,644 DEBUG [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:37,650 DEBUG [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/a/f226f3b6f045455c805696cc1dc74cb5 2024-12-10T20:28:37,650 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] regionserver.HStore(327): Store=48730ca5251b871445d39d47ab457041/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:37,650 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,652 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48730ca5251b871445d39d47ab457041 columnFamilyName b 2024-12-10T20:28:37,652 DEBUG [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:37,652 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] regionserver.HStore(327): Store=48730ca5251b871445d39d47ab457041/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:37,652 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,653 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48730ca5251b871445d39d47ab457041 columnFamilyName c 2024-12-10T20:28:37,654 DEBUG [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:37,660 DEBUG [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/c/bdbb841cca034bf78997757d368834c8 2024-12-10T20:28:37,660 INFO [StoreOpener-48730ca5251b871445d39d47ab457041-1 {}] regionserver.HStore(327): Store=48730ca5251b871445d39d47ab457041/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:37,660 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,661 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,662 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,663 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/recovered.edits/0000000000000000032 2024-12-10T20:28:37,665 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:37,666 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/recovered.edits/0000000000000000032 2024-12-10T20:28:37,666 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 48730ca5251b871445d39d47ab457041 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-10T20:28:37,680 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/.tmp/b/8885a21d207044fbbc66415efae57829 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733862517349/Put/seqid=0 2024-12-10T20:28:37,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741900_1078 (size=5958) 2024-12-10T20:28:37,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741900_1078 (size=5958) 2024-12-10T20:28:37,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741900_1078 (size=5958) 2024-12-10T20:28:37,691 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/.tmp/b/8885a21d207044fbbc66415efae57829 2024-12-10T20:28:37,697 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/.tmp/b/8885a21d207044fbbc66415efae57829 as hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/b/8885a21d207044fbbc66415efae57829 2024-12-10T20:28:37,703 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/b/8885a21d207044fbbc66415efae57829, entries=10, sequenceid=32, filesize=5.8 K 2024-12-10T20:28:37,703 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 48730ca5251b871445d39d47ab457041 in 37ms, sequenceid=32, compaction requested=false; wal=null 2024-12-10T20:28:37,704 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/recovered.edits/0000000000000000032 2024-12-10T20:28:37,706 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,706 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,706 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T20:28:37,708 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 48730ca5251b871445d39d47ab457041 2024-12-10T20:28:37,710 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/48730ca5251b871445d39d47ab457041/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-10T20:28:37,711 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 48730ca5251b871445d39d47ab457041; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64998166, jitterRate=-0.03145185112953186}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T20:28:37,712 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 48730ca5251b871445d39d47ab457041: Writing region info on filesystem at 1733862517642Initializing all the Stores at 1733862517643 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862517643Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862517643Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862517643Obtaining lock to block concurrent updates at 1733862517666 (+23 ms)Preparing flush snapshotting stores in 48730ca5251b871445d39d47ab457041 at 1733862517666Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733862517156.48730ca5251b871445d39d47ab457041., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733862517667 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1733862517156.48730ca5251b871445d39d47ab457041. at 1733862517667Flushing 48730ca5251b871445d39d47ab457041/b: creating writer at 1733862517667Flushing 48730ca5251b871445d39d47ab457041/b: appending metadata at 1733862517679 (+12 ms)Flushing 48730ca5251b871445d39d47ab457041/b: closing flushed file at 1733862517679Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b5cfee7: reopening flushed file at 1733862517696 (+17 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 48730ca5251b871445d39d47ab457041 in 37ms, sequenceid=32, compaction requested=false; wal=null at 1733862517704 (+8 ms)Cleaning up temporary data from old regions at 1733862517706 (+2 ms)Region opened successfully at 1733862517712 (+6 ms) 2024-12-10T20:28:37,736 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=424 (was 414) Potentially hanging thread: PacketResponder: BP-76190982-172.17.0.3-1733862492788:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-76190982-172.17.0.3-1733862492788:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:46466 [Waiting for operation #32] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:57672 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:32978 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-76190982-172.17.0.3-1733862492788:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:32814 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:46566 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:57508 [Waiting for operation #46] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1127 (was 1051) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=306 (was 306), ProcessCount=11 (was 11), AvailableMemoryMB=5099 (was 5127) 2024-12-10T20:28:37,736 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-12-10T20:28:37,749 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=424, OpenFileDescriptor=1127, MaxFileDescriptor=1048576, SystemLoadAverage=306, ProcessCount=11, AvailableMemoryMB=5099 2024-12-10T20:28:37,749 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-12-10T20:28:37,767 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:37,769 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:37,769 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T20:28:37,771 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-74539598, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/hregion-74539598, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:37,781 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-74539598/hregion-74539598.1733862517772, exclude list is [], retry=0 2024-12-10T20:28:37,784 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:37,784 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:37,785 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:37,789 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-74539598/hregion-74539598.1733862517772 2024-12-10T20:28:37,789 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:46745:46745)] 2024-12-10T20:28:37,789 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 411f5391c88606ff72491e9b3fe5637f, NAME => 'testReplayEditsAfterAbortingFlush,,1733862517767.411f5391c88606ff72491e9b3fe5637f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33493/hbase 2024-12-10T20:28:37,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741902_1080 (size=68) 2024-12-10T20:28:37,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741902_1080 (size=68) 2024-12-10T20:28:37,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741902_1080 (size=68) 2024-12-10T20:28:37,799 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733862517767.411f5391c88606ff72491e9b3fe5637f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:37,802 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,803 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 411f5391c88606ff72491e9b3fe5637f columnFamilyName a 2024-12-10T20:28:37,804 DEBUG [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:37,804 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] regionserver.HStore(327): Store=411f5391c88606ff72491e9b3fe5637f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:37,804 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,805 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 411f5391c88606ff72491e9b3fe5637f columnFamilyName b 2024-12-10T20:28:37,805 DEBUG [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:37,806 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] regionserver.HStore(327): Store=411f5391c88606ff72491e9b3fe5637f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:37,806 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,807 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 411f5391c88606ff72491e9b3fe5637f columnFamilyName c 2024-12-10T20:28:37,807 DEBUG [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:37,808 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] regionserver.HStore(327): Store=411f5391c88606ff72491e9b3fe5637f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:37,808 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,809 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,809 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,810 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,810 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,811 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T20:28:37,812 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,814 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:28:37,814 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 411f5391c88606ff72491e9b3fe5637f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74081367, jitterRate=0.10389839112758636}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T20:28:37,815 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 411f5391c88606ff72491e9b3fe5637f: Writing region info on filesystem at 1733862517799Initializing all the Stores at 1733862517800 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862517800Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862517802 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862517802Cleaning up temporary data from old regions at 1733862517810 (+8 ms)Region opened successfully at 1733862517815 (+5 ms) 2024-12-10T20:28:37,815 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 411f5391c88606ff72491e9b3fe5637f, disabling compactions & flushes 2024-12-10T20:28:37,815 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733862517767.411f5391c88606ff72491e9b3fe5637f. 2024-12-10T20:28:37,815 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733862517767.411f5391c88606ff72491e9b3fe5637f. 2024-12-10T20:28:37,815 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733862517767.411f5391c88606ff72491e9b3fe5637f. after waiting 0 ms 2024-12-10T20:28:37,815 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733862517767.411f5391c88606ff72491e9b3fe5637f. 2024-12-10T20:28:37,815 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733862517767.411f5391c88606ff72491e9b3fe5637f. 2024-12-10T20:28:37,815 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 411f5391c88606ff72491e9b3fe5637f: Waiting for close lock at 1733862517815Disabling compacts and flushes for region at 1733862517815Disabling writes for close at 1733862517815Writing region close event to WAL at 1733862517815Closed at 1733862517815 2024-12-10T20:28:37,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741901_1079 (size=93) 2024-12-10T20:28:37,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741901_1079 (size=93) 2024-12-10T20:28:37,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741901_1079 (size=93) 2024-12-10T20:28:37,820 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T20:28:37,820 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-74539598:(num 1733862517772) 2024-12-10T20:28:37,820 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T20:28:37,822 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:37,833 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766/wal.1733862517822, exclude list is [], retry=0 2024-12-10T20:28:37,836 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:37,837 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:37,837 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:37,838 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766/wal.1733862517822 2024-12-10T20:28:37,839 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:40461:40461)] 2024-12-10T20:28:37,889 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 411f5391c88606ff72491e9b3fe5637f, NAME => 'testReplayEditsAfterAbortingFlush,,1733862517767.411f5391c88606ff72491e9b3fe5637f.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:37,891 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,891 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733862517767.411f5391c88606ff72491e9b3fe5637f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:37,891 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,891 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,893 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,894 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 411f5391c88606ff72491e9b3fe5637f columnFamilyName a 2024-12-10T20:28:37,894 DEBUG [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:37,895 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] regionserver.HStore(327): Store=411f5391c88606ff72491e9b3fe5637f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:37,895 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,896 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 411f5391c88606ff72491e9b3fe5637f columnFamilyName b 2024-12-10T20:28:37,896 DEBUG [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:37,897 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] regionserver.HStore(327): Store=411f5391c88606ff72491e9b3fe5637f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:37,897 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,898 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 411f5391c88606ff72491e9b3fe5637f columnFamilyName c 2024-12-10T20:28:37,898 DEBUG [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:37,899 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] regionserver.HStore(327): Store=411f5391c88606ff72491e9b3fe5637f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:37,899 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,900 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,901 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,902 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,902 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,903 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T20:28:37,904 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,905 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 411f5391c88606ff72491e9b3fe5637f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73601598, jitterRate=0.09674927592277527}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T20:28:37,905 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:37,906 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 411f5391c88606ff72491e9b3fe5637f: Running coprocessor pre-open hook at 1733862517891Writing region info on filesystem at 1733862517891Initializing all the Stores at 1733862517892 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862517892Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862517893 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862517893Cleaning up temporary data from old regions at 1733862517902 (+9 ms)Running coprocessor post-open hooks at 1733862517905 (+3 ms)Region opened successfully at 1733862517906 (+1 ms) 2024-12-10T20:28:37,921 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 411f5391c88606ff72491e9b3fe5637f 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-10T20:28:37,922 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 411f5391c88606ff72491e9b3fe5637f/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:38,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-10T20:28:38,123 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-10T20:28:38,124 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-10T20:28:38,124 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-10T20:28:38,125 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-10T20:28:38,125 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-10T20:28:38,126 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-10T20:28:38,126 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-10T20:28:38,126 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-10T20:28:38,126 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-10T20:28:38,923 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 411f5391c88606ff72491e9b3fe5637f/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:39,573 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T20:28:39,925 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 411f5391c88606ff72491e9b3fe5637f/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:40,926 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 411f5391c88606ff72491e9b3fe5637f/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:41,927 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 411f5391c88606ff72491e9b3fe5637f/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:42,928 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 411f5391c88606ff72491e9b3fe5637f/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:43,930 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 411f5391c88606ff72491e9b3fe5637f/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:44,930 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 411f5391c88606ff72491e9b3fe5637f/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:45,931 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 411f5391c88606ff72491e9b3fe5637f/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:46,932 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 411f5391c88606ff72491e9b3fe5637f/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:46,933 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 411f5391c88606ff72491e9b3fe5637f: 2024-12-10T20:28:46,933 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733862517767.411f5391c88606ff72491e9b3fe5637f. 2024-12-10T20:28:46,948 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 411f5391c88606ff72491e9b3fe5637f: 2024-12-10T20:28:46,948 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-10T20:28:46,949 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 411f5391c88606ff72491e9b3fe5637f, disabling compactions & flushes 2024-12-10T20:28:46,949 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733862517767.411f5391c88606ff72491e9b3fe5637f. 2024-12-10T20:28:46,949 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733862517767.411f5391c88606ff72491e9b3fe5637f. 2024-12-10T20:28:46,949 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733862517767.411f5391c88606ff72491e9b3fe5637f. after waiting 0 ms 2024-12-10T20:28:46,949 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733862517767.411f5391c88606ff72491e9b3fe5637f. 2024-12-10T20:28:46,949 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733862517767.411f5391c88606ff72491e9b3fe5637f. 2024-12-10T20:28:46,949 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733862517767.411f5391c88606ff72491e9b3fe5637f. 2024-12-10T20:28:46,949 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 411f5391c88606ff72491e9b3fe5637f: Waiting for close lock at 1733862526949Running coprocessor pre-close hooks at 1733862526949Disabling compacts and flushes for region at 1733862526949Disabling writes for close at 1733862526949Writing region close event to WAL at 1733862526949Running coprocessor post-close hooks at 1733862526949Closed at 1733862526949 2024-12-10T20:28:46,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741903_1081 (size=2398) 2024-12-10T20:28:46,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741903_1081 (size=2398) 2024-12-10T20:28:46,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741903_1081 (size=2398) 2024-12-10T20:28:46,962 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-10T20:28:46,973 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33493/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766/wal.1733862517822, size=2.3 K (2398bytes) 2024-12-10T20:28:46,973 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33493/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766/wal.1733862517822 2024-12-10T20:28:46,973 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33493/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766/wal.1733862517822 after 0ms 2024-12-10T20:28:46,976 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766/wal.1733862517822: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:46,976 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33493/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766/wal.1733862517822 took 3ms 2024-12-10T20:28:46,979 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33493/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766/wal.1733862517822 so closing down 2024-12-10T20:28:46,979 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T20:28:46,980 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733862517822.temp 2024-12-10T20:28:46,981 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/recovered.edits/0000000000000000004-wal.1733862517822.temp 2024-12-10T20:28:46,981 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T20:28:46,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741904_1082 (size=1672) 2024-12-10T20:28:46,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741904_1082 (size=1672) 2024-12-10T20:28:46,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741904_1082 (size=1672) 2024-12-10T20:28:46,993 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/recovered.edits/0000000000000000004-wal.1733862517822.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-10T20:28:46,994 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/recovered.edits/0000000000000000004-wal.1733862517822.temp to hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/recovered.edits/0000000000000000026 2024-12-10T20:28:46,995 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:33493/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766/wal.1733862517822, size=2.3 K, length=2398, corrupted=false, cancelled=false 2024-12-10T20:28:46,995 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33493/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766/wal.1733862517822, journal: Splitting hdfs://localhost:33493/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766/wal.1733862517822, size=2.3 K (2398bytes) at 1733862526973Finishing writing output for hdfs://localhost:33493/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766/wal.1733862517822 so closing down at 1733862526979 (+6 ms)Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/recovered.edits/0000000000000000004-wal.1733862517822.temp at 1733862526981 (+2 ms)3 split writer threads finished at 1733862526981Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/recovered.edits/0000000000000000004-wal.1733862517822.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733862526993 (+12 ms)Rename recovered edits hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/recovered.edits/0000000000000000004-wal.1733862517822.temp to hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/recovered.edits/0000000000000000026 at 1733862526994 (+1 ms)Processed 23 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:33493/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766/wal.1733862517822, size=2.3 K, length=2398, corrupted=false, cancelled=false at 1733862526995 (+1 ms) 2024-12-10T20:28:46,996 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33493/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766/wal.1733862517822 to hdfs://localhost:33493/hbase/oldWALs/wal.1733862517822 2024-12-10T20:28:46,997 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/recovered.edits/0000000000000000026 2024-12-10T20:28:46,997 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T20:28:46,999 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:47,010 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766/wal.1733862527000, exclude list is [], retry=0 2024-12-10T20:28:47,013 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:47,013 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:47,013 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:47,015 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766/wal.1733862527000 2024-12-10T20:28:47,015 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:33825:33825)] 2024-12-10T20:28:47,015 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 411f5391c88606ff72491e9b3fe5637f, NAME => 'testReplayEditsAfterAbortingFlush,,1733862517767.411f5391c88606ff72491e9b3fe5637f.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:47,016 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:47,016 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733862517767.411f5391c88606ff72491e9b3fe5637f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:47,016 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:47,016 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:47,018 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:47,019 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 411f5391c88606ff72491e9b3fe5637f columnFamilyName a 2024-12-10T20:28:47,019 DEBUG [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:47,019 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] regionserver.HStore(327): Store=411f5391c88606ff72491e9b3fe5637f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:47,020 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:47,020 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 411f5391c88606ff72491e9b3fe5637f columnFamilyName b 2024-12-10T20:28:47,020 DEBUG [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:47,021 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] regionserver.HStore(327): Store=411f5391c88606ff72491e9b3fe5637f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:47,021 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:47,022 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 411f5391c88606ff72491e9b3fe5637f columnFamilyName c 2024-12-10T20:28:47,022 DEBUG [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:47,023 INFO [StoreOpener-411f5391c88606ff72491e9b3fe5637f-1 {}] regionserver.HStore(327): Store=411f5391c88606ff72491e9b3fe5637f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:47,023 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:47,024 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:47,025 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:47,026 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/recovered.edits/0000000000000000026 2024-12-10T20:28:47,028 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:47,029 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/recovered.edits/0000000000000000026 2024-12-10T20:28:47,029 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 411f5391c88606ff72491e9b3fe5637f 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-10T20:28:47,041 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/.tmp/a/604eff83252a4f0aac8968cd3b14bd10 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733862526938/Put/seqid=0 2024-12-10T20:28:47,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741906_1084 (size=5523) 2024-12-10T20:28:47,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741906_1084 (size=5523) 2024-12-10T20:28:47,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741906_1084 (size=5523) 2024-12-10T20:28:47,047 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/.tmp/a/604eff83252a4f0aac8968cd3b14bd10 2024-12-10T20:28:47,064 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/.tmp/b/f4eaddcc0fb247b4b4c8055f856bf6f7 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733862526933/Put/seqid=0 2024-12-10T20:28:47,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741907_1085 (size=5524) 2024-12-10T20:28:47,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741907_1085 (size=5524) 2024-12-10T20:28:47,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741907_1085 (size=5524) 2024-12-10T20:28:47,071 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/.tmp/b/f4eaddcc0fb247b4b4c8055f856bf6f7 2024-12-10T20:28:47,090 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/.tmp/c/4ce1480545c84d36b1929b684700e277 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733862526935/Put/seqid=0 2024-12-10T20:28:47,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741908_1086 (size=5457) 2024-12-10T20:28:47,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741908_1086 (size=5457) 2024-12-10T20:28:47,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741908_1086 (size=5457) 2024-12-10T20:28:47,096 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/.tmp/c/4ce1480545c84d36b1929b684700e277 2024-12-10T20:28:47,102 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/.tmp/a/604eff83252a4f0aac8968cd3b14bd10 as hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/a/604eff83252a4f0aac8968cd3b14bd10 2024-12-10T20:28:47,108 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/a/604eff83252a4f0aac8968cd3b14bd10, entries=7, sequenceid=26, filesize=5.4 K 2024-12-10T20:28:47,109 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/.tmp/b/f4eaddcc0fb247b4b4c8055f856bf6f7 as hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/b/f4eaddcc0fb247b4b4c8055f856bf6f7 2024-12-10T20:28:47,114 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/b/f4eaddcc0fb247b4b4c8055f856bf6f7, entries=7, sequenceid=26, filesize=5.4 K 2024-12-10T20:28:47,115 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/.tmp/c/4ce1480545c84d36b1929b684700e277 as hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/c/4ce1480545c84d36b1929b684700e277 2024-12-10T20:28:47,121 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/c/4ce1480545c84d36b1929b684700e277, entries=6, sequenceid=26, filesize=5.3 K 2024-12-10T20:28:47,121 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 411f5391c88606ff72491e9b3fe5637f in 92ms, sequenceid=26, compaction requested=false; wal=null 2024-12-10T20:28:47,122 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/recovered.edits/0000000000000000026 2024-12-10T20:28:47,123 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:47,123 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:47,124 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T20:28:47,125 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:47,127 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testReplayEditsAfterAbortingFlush/411f5391c88606ff72491e9b3fe5637f/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-10T20:28:47,128 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 411f5391c88606ff72491e9b3fe5637f; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60029224, jitterRate=-0.10549485683441162}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T20:28:47,128 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 411f5391c88606ff72491e9b3fe5637f 2024-12-10T20:28:47,129 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 411f5391c88606ff72491e9b3fe5637f: Running coprocessor pre-open hook at 1733862527016Writing region info on filesystem at 1733862527016Initializing all the Stores at 1733862527017 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862527017Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862527017Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862527017Obtaining lock to block concurrent updates at 1733862527029 (+12 ms)Preparing flush snapshotting stores in 411f5391c88606ff72491e9b3fe5637f at 1733862527029Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733862517767.411f5391c88606ff72491e9b3fe5637f., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733862527029Flushing stores of testReplayEditsAfterAbortingFlush,,1733862517767.411f5391c88606ff72491e9b3fe5637f. at 1733862527029Flushing 411f5391c88606ff72491e9b3fe5637f/a: creating writer at 1733862527029Flushing 411f5391c88606ff72491e9b3fe5637f/a: appending metadata at 1733862527041 (+12 ms)Flushing 411f5391c88606ff72491e9b3fe5637f/a: closing flushed file at 1733862527041Flushing 411f5391c88606ff72491e9b3fe5637f/b: creating writer at 1733862527051 (+10 ms)Flushing 411f5391c88606ff72491e9b3fe5637f/b: appending metadata at 1733862527064 (+13 ms)Flushing 411f5391c88606ff72491e9b3fe5637f/b: closing flushed file at 1733862527064Flushing 411f5391c88606ff72491e9b3fe5637f/c: creating writer at 1733862527075 (+11 ms)Flushing 411f5391c88606ff72491e9b3fe5637f/c: appending metadata at 1733862527089 (+14 ms)Flushing 411f5391c88606ff72491e9b3fe5637f/c: closing flushed file at 1733862527090 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f1c72c4: reopening flushed file at 1733862527101 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59b3981e: reopening flushed file at 1733862527108 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b036e1d: reopening flushed file at 1733862527114 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 411f5391c88606ff72491e9b3fe5637f in 92ms, sequenceid=26, compaction requested=false; wal=null at 1733862527121 (+7 ms)Cleaning up temporary data from old regions at 1733862527123 (+2 ms)Running coprocessor post-open hooks at 1733862527128 (+5 ms)Region opened successfully at 1733862527128 2024-12-10T20:28:47,146 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=424 (was 424), OpenFileDescriptor=1183 (was 1127) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=281 (was 306), ProcessCount=11 (was 11), AvailableMemoryMB=6107 (was 5099) - AvailableMemoryMB LEAK? - 2024-12-10T20:28:47,146 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-12-10T20:28:47,156 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=424, OpenFileDescriptor=1183, MaxFileDescriptor=1048576, SystemLoadAverage=281, ProcessCount=11, AvailableMemoryMB=6107 2024-12-10T20:28:47,156 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-12-10T20:28:47,169 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:47,170 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:47,170 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T20:28:47,173 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-09512036, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/hregion-09512036, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:47,183 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-09512036/hregion-09512036.1733862527173, exclude list is [], retry=0 2024-12-10T20:28:47,185 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:47,185 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:47,186 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:47,188 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-09512036/hregion-09512036.1733862527173 2024-12-10T20:28:47,188 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:46745:46745)] 2024-12-10T20:28:47,188 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a0350f7b7ec66d6a0b04bc478bca295e, NAME => 'testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33493/hbase 2024-12-10T20:28:47,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741910_1088 (size=61) 2024-12-10T20:28:47,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741910_1088 (size=61) 2024-12-10T20:28:47,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741910_1088 (size=61) 2024-12-10T20:28:47,197 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:47,198 INFO [StoreOpener-a0350f7b7ec66d6a0b04bc478bca295e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,200 INFO [StoreOpener-a0350f7b7ec66d6a0b04bc478bca295e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0350f7b7ec66d6a0b04bc478bca295e columnFamilyName a 2024-12-10T20:28:47,200 DEBUG [StoreOpener-a0350f7b7ec66d6a0b04bc478bca295e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:47,200 INFO [StoreOpener-a0350f7b7ec66d6a0b04bc478bca295e-1 {}] regionserver.HStore(327): Store=a0350f7b7ec66d6a0b04bc478bca295e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:47,200 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,201 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,201 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,201 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,201 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,203 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,205 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:28:47,206 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a0350f7b7ec66d6a0b04bc478bca295e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72082289, jitterRate=0.07410980761051178}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T20:28:47,207 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a0350f7b7ec66d6a0b04bc478bca295e: Writing region info on filesystem at 1733862527197Initializing all the Stores at 1733862527198 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862527198Cleaning up temporary data from old regions at 1733862527201 (+3 ms)Region opened successfully at 1733862527206 (+5 ms) 2024-12-10T20:28:47,207 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a0350f7b7ec66d6a0b04bc478bca295e, disabling compactions & flushes 2024-12-10T20:28:47,207 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e. 2024-12-10T20:28:47,207 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e. 2024-12-10T20:28:47,207 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e. after waiting 0 ms 2024-12-10T20:28:47,207 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e. 2024-12-10T20:28:47,209 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e. 2024-12-10T20:28:47,209 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a0350f7b7ec66d6a0b04bc478bca295e: Waiting for close lock at 1733862527207Disabling compacts and flushes for region at 1733862527207Disabling writes for close at 1733862527207Writing region close event to WAL at 1733862527209 (+2 ms)Closed at 1733862527209 2024-12-10T20:28:47,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741909_1087 (size=93) 2024-12-10T20:28:47,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741909_1087 (size=93) 2024-12-10T20:28:47,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741909_1087 (size=93) 2024-12-10T20:28:47,216 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T20:28:47,216 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-09512036:(num 1733862527173) 2024-12-10T20:28:47,216 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T20:28:47,218 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:47,230 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168/wal.1733862527218, exclude list is [], retry=0 2024-12-10T20:28:47,233 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:47,233 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:47,233 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:47,238 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168/wal.1733862527218 2024-12-10T20:28:47,238 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:40461:40461)] 2024-12-10T20:28:47,239 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a0350f7b7ec66d6a0b04bc478bca295e, NAME => 'testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:47,239 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:47,239 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,239 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,240 INFO [StoreOpener-a0350f7b7ec66d6a0b04bc478bca295e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,241 INFO [StoreOpener-a0350f7b7ec66d6a0b04bc478bca295e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0350f7b7ec66d6a0b04bc478bca295e columnFamilyName a 2024-12-10T20:28:47,241 DEBUG [StoreOpener-a0350f7b7ec66d6a0b04bc478bca295e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:47,242 INFO [StoreOpener-a0350f7b7ec66d6a0b04bc478bca295e-1 {}] regionserver.HStore(327): Store=a0350f7b7ec66d6a0b04bc478bca295e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:47,242 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,243 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,245 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,245 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,245 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,248 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,248 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a0350f7b7ec66d6a0b04bc478bca295e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71846283, jitterRate=0.07059304416179657}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T20:28:47,249 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a0350f7b7ec66d6a0b04bc478bca295e: Writing region info on filesystem at 1733862527239Initializing all the Stores at 1733862527240 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862527240Cleaning up temporary data from old regions at 1733862527245 (+5 ms)Region opened successfully at 1733862527249 (+4 ms) 2024-12-10T20:28:47,263 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a0350f7b7ec66d6a0b04bc478bca295e, disabling compactions & flushes 2024-12-10T20:28:47,263 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e. 2024-12-10T20:28:47,263 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e. 2024-12-10T20:28:47,263 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e. after waiting 0 ms 2024-12-10T20:28:47,263 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e. 2024-12-10T20:28:47,265 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e. 2024-12-10T20:28:47,265 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e. 2024-12-10T20:28:47,265 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a0350f7b7ec66d6a0b04bc478bca295e: Waiting for close lock at 1733862527263Disabling compacts and flushes for region at 1733862527263Disabling writes for close at 1733862527263Writing region close event to WAL at 1733862527265 (+2 ms)Closed at 1733862527265 2024-12-10T20:28:47,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741911_1089 (size=838) 2024-12-10T20:28:47,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741911_1089 (size=838) 2024-12-10T20:28:47,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741911_1089 (size=838) 2024-12-10T20:28:47,285 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33493/hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168/wal.1733862527218, size=838 (838bytes) 2024-12-10T20:28:47,285 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33493/hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168/wal.1733862527218 2024-12-10T20:28:47,286 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33493/hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168/wal.1733862527218 after 0ms 2024-12-10T20:28:47,288 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168/wal.1733862527218: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:47,288 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33493/hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168/wal.1733862527218 took 3ms 2024-12-10T20:28:47,290 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33493/hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168/wal.1733862527218 so closing down 2024-12-10T20:28:47,291 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T20:28:47,294 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733862527218.temp 2024-12-10T20:28:47,295 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/recovered.edits/0000000000000000003-wal.1733862527218.temp 2024-12-10T20:28:47,295 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T20:28:47,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741912_1090 (size=838) 2024-12-10T20:28:47,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741912_1090 (size=838) 2024-12-10T20:28:47,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741912_1090 (size=838) 2024-12-10T20:28:47,304 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/recovered.edits/0000000000000000003-wal.1733862527218.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-10T20:28:47,305 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/recovered.edits/0000000000000000003-wal.1733862527218.temp to hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/recovered.edits/0000000000000000012 2024-12-10T20:28:47,306 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:33493/hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168/wal.1733862527218, size=838, length=838, corrupted=false, cancelled=false 2024-12-10T20:28:47,306 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33493/hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168/wal.1733862527218, journal: Splitting hdfs://localhost:33493/hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168/wal.1733862527218, size=838 (838bytes) at 1733862527285Finishing writing output for hdfs://localhost:33493/hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168/wal.1733862527218 so closing down at 1733862527290 (+5 ms)Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/recovered.edits/0000000000000000003-wal.1733862527218.temp at 1733862527295 (+5 ms)3 split writer threads finished at 1733862527295Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/recovered.edits/0000000000000000003-wal.1733862527218.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733862527304 (+9 ms)Rename recovered edits hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/recovered.edits/0000000000000000003-wal.1733862527218.temp to hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/recovered.edits/0000000000000000012 at 1733862527305 (+1 ms)Processed 10 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:33493/hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168/wal.1733862527218, size=838, length=838, corrupted=false, cancelled=false at 1733862527306 (+1 ms) 2024-12-10T20:28:47,308 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33493/hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168/wal.1733862527218 to hdfs://localhost:33493/hbase/oldWALs/wal.1733862527218 2024-12-10T20:28:47,309 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/recovered.edits/0000000000000000012 2024-12-10T20:28:47,313 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:47,814 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T20:28:47,817 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:47,837 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168/wal.1733862527818, exclude list is [], retry=0 2024-12-10T20:28:47,840 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:47,840 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:47,840 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:47,853 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168/wal.1733862527818 2024-12-10T20:28:47,853 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:33825:33825)] 2024-12-10T20:28:47,853 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a0350f7b7ec66d6a0b04bc478bca295e, NAME => 'testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:47,854 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:47,854 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,854 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,861 INFO [StoreOpener-a0350f7b7ec66d6a0b04bc478bca295e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,863 INFO [StoreOpener-a0350f7b7ec66d6a0b04bc478bca295e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0350f7b7ec66d6a0b04bc478bca295e columnFamilyName a 2024-12-10T20:28:47,863 DEBUG [StoreOpener-a0350f7b7ec66d6a0b04bc478bca295e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:47,869 INFO [StoreOpener-a0350f7b7ec66d6a0b04bc478bca295e-1 {}] regionserver.HStore(327): Store=a0350f7b7ec66d6a0b04bc478bca295e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:47,869 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,870 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,874 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,874 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/recovered.edits/0000000000000000012 2024-12-10T20:28:47,878 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:47,879 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/recovered.edits/0000000000000000012 2024-12-10T20:28:47,880 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a0350f7b7ec66d6a0b04bc478bca295e 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-10T20:28:47,909 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/.tmp/a/c9f6d0e2bcb6417c96b69596c04e8aff is 79, key is testDatalossWhenInputError/a:x0/1733862527249/Put/seqid=0 2024-12-10T20:28:47,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741914_1092 (size=5808) 2024-12-10T20:28:47,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741914_1092 (size=5808) 2024-12-10T20:28:47,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741914_1092 (size=5808) 2024-12-10T20:28:47,932 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/.tmp/a/c9f6d0e2bcb6417c96b69596c04e8aff 2024-12-10T20:28:47,954 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/.tmp/a/c9f6d0e2bcb6417c96b69596c04e8aff as hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/a/c9f6d0e2bcb6417c96b69596c04e8aff 2024-12-10T20:28:47,966 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/a/c9f6d0e2bcb6417c96b69596c04e8aff, entries=10, sequenceid=12, filesize=5.7 K 2024-12-10T20:28:47,967 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for a0350f7b7ec66d6a0b04bc478bca295e in 87ms, sequenceid=12, compaction requested=false; wal=null 2024-12-10T20:28:47,967 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/recovered.edits/0000000000000000012 2024-12-10T20:28:47,969 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,969 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,973 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,979 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-10T20:28:47,981 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a0350f7b7ec66d6a0b04bc478bca295e; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67817974, jitterRate=0.010566562414169312}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T20:28:47,981 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a0350f7b7ec66d6a0b04bc478bca295e: Writing region info on filesystem at 1733862527854Initializing all the Stores at 1733862527856 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862527856Obtaining lock to block concurrent updates at 1733862527880 (+24 ms)Preparing flush snapshotting stores in a0350f7b7ec66d6a0b04bc478bca295e at 1733862527880Finished memstore snapshotting testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733862527880Flushing stores of testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e. at 1733862527880Flushing a0350f7b7ec66d6a0b04bc478bca295e/a: creating writer at 1733862527880Flushing a0350f7b7ec66d6a0b04bc478bca295e/a: appending metadata at 1733862527907 (+27 ms)Flushing a0350f7b7ec66d6a0b04bc478bca295e/a: closing flushed file at 1733862527908 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@658b95d8: reopening flushed file at 1733862527946 (+38 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for a0350f7b7ec66d6a0b04bc478bca295e in 87ms, sequenceid=12, compaction requested=false; wal=null at 1733862527967 (+21 ms)Cleaning up temporary data from old regions at 1733862527969 (+2 ms)Region opened successfully at 1733862527981 (+12 ms) 2024-12-10T20:28:47,985 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a0350f7b7ec66d6a0b04bc478bca295e, NAME => 'testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:47,985 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733862527169.a0350f7b7ec66d6a0b04bc478bca295e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:47,985 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,985 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,987 INFO [StoreOpener-a0350f7b7ec66d6a0b04bc478bca295e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,988 INFO [StoreOpener-a0350f7b7ec66d6a0b04bc478bca295e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0350f7b7ec66d6a0b04bc478bca295e columnFamilyName a 2024-12-10T20:28:47,988 DEBUG [StoreOpener-a0350f7b7ec66d6a0b04bc478bca295e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:47,997 DEBUG [StoreOpener-a0350f7b7ec66d6a0b04bc478bca295e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/a/c9f6d0e2bcb6417c96b69596c04e8aff 2024-12-10T20:28:47,997 INFO [StoreOpener-a0350f7b7ec66d6a0b04bc478bca295e-1 {}] regionserver.HStore(327): Store=a0350f7b7ec66d6a0b04bc478bca295e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:47,997 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,998 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,999 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,999 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:47,999 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:48,001 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a0350f7b7ec66d6a0b04bc478bca295e 2024-12-10T20:28:48,003 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testDatalossWhenInputError/a0350f7b7ec66d6a0b04bc478bca295e/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-10T20:28:48,004 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a0350f7b7ec66d6a0b04bc478bca295e; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67834820, jitterRate=0.01081758737564087}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T20:28:48,005 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a0350f7b7ec66d6a0b04bc478bca295e: Writing region info on filesystem at 1733862527985Initializing all the Stores at 1733862527986 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862527986Cleaning up temporary data from old regions at 1733862527999 (+13 ms)Region opened successfully at 1733862528005 (+6 ms) 2024-12-10T20:28:48,036 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=433 (was 424) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:40588 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-76190982-172.17.0.3-1733862492788:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:47900 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-76190982-172.17.0.3-1733862492788:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:48100 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:48172 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:47938 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-76190982-172.17.0.3-1733862492788:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1183) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=299 (was 281) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5977 (was 6107) 2024-12-10T20:28:48,036 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-10T20:28:48,051 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=433, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=299, ProcessCount=11, AvailableMemoryMB=5968 2024-12-10T20:28:48,051 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-10T20:28:48,068 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:48,070 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:48,071 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T20:28:48,073 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-99523026, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/hregion-99523026, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:48,091 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-99523026/hregion-99523026.1733862528074, exclude list is [], retry=0 2024-12-10T20:28:48,095 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:48,097 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:48,098 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:48,109 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-99523026/hregion-99523026.1733862528074 2024-12-10T20:28:48,110 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:46745:46745)] 2024-12-10T20:28:48,110 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6b0091041a95efa38bb4abdd89c08422, NAME => 'testCompactedBulkLoadedFiles,,1733862528068.6b0091041a95efa38bb4abdd89c08422.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33493/hbase 2024-12-10T20:28:48,123 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-10T20:28:48,123 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-10T20:28:48,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741916_1094 (size=63) 2024-12-10T20:28:48,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741916_1094 (size=63) 2024-12-10T20:28:48,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741916_1094 (size=63) 2024-12-10T20:28:48,135 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733862528068.6b0091041a95efa38bb4abdd89c08422.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:48,141 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:48,142 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b0091041a95efa38bb4abdd89c08422 columnFamilyName a 2024-12-10T20:28:48,142 DEBUG [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:48,143 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.HStore(327): Store=6b0091041a95efa38bb4abdd89c08422/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:48,143 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:48,144 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b0091041a95efa38bb4abdd89c08422 columnFamilyName b 2024-12-10T20:28:48,144 DEBUG [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:48,145 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.HStore(327): Store=6b0091041a95efa38bb4abdd89c08422/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:48,145 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:48,146 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b0091041a95efa38bb4abdd89c08422 columnFamilyName c 2024-12-10T20:28:48,146 DEBUG [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:48,147 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.HStore(327): Store=6b0091041a95efa38bb4abdd89c08422/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:48,147 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:48,148 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:48,148 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:48,149 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:48,149 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:48,149 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T20:28:48,150 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:48,152 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:28:48,153 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6b0091041a95efa38bb4abdd89c08422; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65099172, jitterRate=-0.029946744441986084}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T20:28:48,153 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6b0091041a95efa38bb4abdd89c08422: Writing region info on filesystem at 1733862528135Initializing all the Stores at 1733862528136 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862528136Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862528141 (+5 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862528141Cleaning up temporary data from old regions at 1733862528149 (+8 ms)Region opened successfully at 1733862528153 (+4 ms) 2024-12-10T20:28:48,153 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6b0091041a95efa38bb4abdd89c08422, disabling compactions & flushes 2024-12-10T20:28:48,153 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733862528068.6b0091041a95efa38bb4abdd89c08422. 2024-12-10T20:28:48,153 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733862528068.6b0091041a95efa38bb4abdd89c08422. 2024-12-10T20:28:48,153 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733862528068.6b0091041a95efa38bb4abdd89c08422. after waiting 0 ms 2024-12-10T20:28:48,153 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733862528068.6b0091041a95efa38bb4abdd89c08422. 2024-12-10T20:28:48,154 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733862528068.6b0091041a95efa38bb4abdd89c08422. 2024-12-10T20:28:48,154 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6b0091041a95efa38bb4abdd89c08422: Waiting for close lock at 1733862528153Disabling compacts and flushes for region at 1733862528153Disabling writes for close at 1733862528153Writing region close event to WAL at 1733862528154 (+1 ms)Closed at 1733862528154 2024-12-10T20:28:48,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741915_1093 (size=93) 2024-12-10T20:28:48,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741915_1093 (size=93) 2024-12-10T20:28:48,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741915_1093 (size=93) 2024-12-10T20:28:48,159 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T20:28:48,159 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-99523026:(num 1733862528074) 2024-12-10T20:28:48,159 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T20:28:48,161 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:48,178 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862528161, exclude list is [], retry=0 2024-12-10T20:28:48,181 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:48,182 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:48,182 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:48,186 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862528161 2024-12-10T20:28:48,186 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:33825:33825)] 2024-12-10T20:28:48,186 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6b0091041a95efa38bb4abdd89c08422, NAME => 'testCompactedBulkLoadedFiles,,1733862528068.6b0091041a95efa38bb4abdd89c08422.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:48,186 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733862528068.6b0091041a95efa38bb4abdd89c08422.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:48,186 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:48,186 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:48,188 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:48,189 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b0091041a95efa38bb4abdd89c08422 columnFamilyName a 2024-12-10T20:28:48,189 DEBUG [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:48,189 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.HStore(327): Store=6b0091041a95efa38bb4abdd89c08422/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:48,189 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:48,190 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b0091041a95efa38bb4abdd89c08422 columnFamilyName b 2024-12-10T20:28:48,190 DEBUG [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:48,191 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.HStore(327): Store=6b0091041a95efa38bb4abdd89c08422/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:48,191 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:48,192 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b0091041a95efa38bb4abdd89c08422 columnFamilyName c 2024-12-10T20:28:48,192 DEBUG [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:48,192 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.HStore(327): Store=6b0091041a95efa38bb4abdd89c08422/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:48,192 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:48,193 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:48,194 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:48,195 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:48,195 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:48,195 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T20:28:48,197 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:48,198 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6b0091041a95efa38bb4abdd89c08422; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63830459, jitterRate=-0.048852041363716125}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T20:28:48,198 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6b0091041a95efa38bb4abdd89c08422: Writing region info on filesystem at 1733862528186Initializing all the Stores at 1733862528187 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862528187Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862528187Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862528187Cleaning up temporary data from old regions at 1733862528195 (+8 ms)Region opened successfully at 1733862528198 (+3 ms) 2024-12-10T20:28:48,203 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733862528203/Put/seqid=0 2024-12-10T20:28:48,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741918_1096 (size=4875) 2024-12-10T20:28:48,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741918_1096 (size=4875) 2024-12-10T20:28:48,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741918_1096 (size=4875) 2024-12-10T20:28:48,211 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733862528211/Put/seqid=0 2024-12-10T20:28:48,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741919_1097 (size=4875) 2024-12-10T20:28:48,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741919_1097 (size=4875) 2024-12-10T20:28:48,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741919_1097 (size=4875) 2024-12-10T20:28:48,220 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733862528220/Put/seqid=0 2024-12-10T20:28:48,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741920_1098 (size=4875) 2024-12-10T20:28:48,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741920_1098 (size=4875) 2024-12-10T20:28:48,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741920_1098 (size=4875) 2024-12-10T20:28:48,226 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33493/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 6b0091041a95efa38bb4abdd89c08422/a 2024-12-10T20:28:48,230 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-10T20:28:48,230 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-10T20:28:48,230 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33493/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 6b0091041a95efa38bb4abdd89c08422/a 2024-12-10T20:28:48,233 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-10T20:28:48,233 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-10T20:28:48,233 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33493/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 6b0091041a95efa38bb4abdd89c08422/a 2024-12-10T20:28:48,237 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-10T20:28:48,237 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-10T20:28:48,237 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6b0091041a95efa38bb4abdd89c08422 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-10T20:28:48,252 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/.tmp/a/68237c2b1cb042eda86ea80680a24011 is 55, key is testCompactedBulkLoadedFiles/a:a/1733862528199/Put/seqid=0 2024-12-10T20:28:48,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741921_1099 (size=5107) 2024-12-10T20:28:48,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741921_1099 (size=5107) 2024-12-10T20:28:48,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741921_1099 (size=5107) 2024-12-10T20:28:48,261 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/.tmp/a/68237c2b1cb042eda86ea80680a24011 2024-12-10T20:28:48,265 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/.tmp/a/68237c2b1cb042eda86ea80680a24011 as hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/68237c2b1cb042eda86ea80680a24011 2024-12-10T20:28:48,270 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/68237c2b1cb042eda86ea80680a24011, entries=1, sequenceid=4, filesize=5.0 K 2024-12-10T20:28:48,271 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 6b0091041a95efa38bb4abdd89c08422 in 34ms, sequenceid=4, compaction requested=false 2024-12-10T20:28:48,272 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6b0091041a95efa38bb4abdd89c08422: 2024-12-10T20:28:48,273 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/5d82ac8f545a46799c95f8a1ec91811b_SeqId_4_ 2024-12-10T20:28:48,275 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/eeda043a3403485eb0ed93168217a78d_SeqId_4_ 2024-12-10T20:28:48,276 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/38b327ea9be94a8ebe7ee8cb47af9868_SeqId_4_ 2024-12-10T20:28:48,277 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33493/hbase/testCompactedBulkLoadedFiles/hfile0 into 6b0091041a95efa38bb4abdd89c08422/a as hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/5d82ac8f545a46799c95f8a1ec91811b_SeqId_4_ - updating store file list. 2024-12-10T20:28:48,283 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5d82ac8f545a46799c95f8a1ec91811b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-10T20:28:48,283 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/5d82ac8f545a46799c95f8a1ec91811b_SeqId_4_ into 6b0091041a95efa38bb4abdd89c08422/a 2024-12-10T20:28:48,283 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33493/hbase/testCompactedBulkLoadedFiles/hfile0 into 6b0091041a95efa38bb4abdd89c08422/a (new location: hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/5d82ac8f545a46799c95f8a1ec91811b_SeqId_4_) 2024-12-10T20:28:48,285 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33493/hbase/testCompactedBulkLoadedFiles/hfile1 into 6b0091041a95efa38bb4abdd89c08422/a as hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/eeda043a3403485eb0ed93168217a78d_SeqId_4_ - updating store file list. 2024-12-10T20:28:48,291 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for eeda043a3403485eb0ed93168217a78d_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-10T20:28:48,291 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/eeda043a3403485eb0ed93168217a78d_SeqId_4_ into 6b0091041a95efa38bb4abdd89c08422/a 2024-12-10T20:28:48,291 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33493/hbase/testCompactedBulkLoadedFiles/hfile1 into 6b0091041a95efa38bb4abdd89c08422/a (new location: hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/eeda043a3403485eb0ed93168217a78d_SeqId_4_) 2024-12-10T20:28:48,293 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33493/hbase/testCompactedBulkLoadedFiles/hfile2 into 6b0091041a95efa38bb4abdd89c08422/a as hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/38b327ea9be94a8ebe7ee8cb47af9868_SeqId_4_ - updating store file list. 2024-12-10T20:28:48,299 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 38b327ea9be94a8ebe7ee8cb47af9868_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-10T20:28:48,299 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/38b327ea9be94a8ebe7ee8cb47af9868_SeqId_4_ into 6b0091041a95efa38bb4abdd89c08422/a 2024-12-10T20:28:48,299 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33493/hbase/testCompactedBulkLoadedFiles/hfile2 into 6b0091041a95efa38bb4abdd89c08422/a (new location: hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/38b327ea9be94a8ebe7ee8cb47af9868_SeqId_4_) 2024-12-10T20:28:48,309 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-10T20:28:48,309 DEBUG [Time-limited test {}] regionserver.HStore(1541): 6b0091041a95efa38bb4abdd89c08422/a is initiating major compaction (all files) 2024-12-10T20:28:48,309 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 6b0091041a95efa38bb4abdd89c08422/a in testCompactedBulkLoadedFiles,,1733862528068.6b0091041a95efa38bb4abdd89c08422. 2024-12-10T20:28:48,309 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/68237c2b1cb042eda86ea80680a24011, hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/5d82ac8f545a46799c95f8a1ec91811b_SeqId_4_, hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/eeda043a3403485eb0ed93168217a78d_SeqId_4_, hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/38b327ea9be94a8ebe7ee8cb47af9868_SeqId_4_] into tmpdir=hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/.tmp, totalSize=19.3 K 2024-12-10T20:28:48,310 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 68237c2b1cb042eda86ea80680a24011, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733862528199 2024-12-10T20:28:48,311 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 5d82ac8f545a46799c95f8a1ec91811b_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-10T20:28:48,311 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting eeda043a3403485eb0ed93168217a78d_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-10T20:28:48,312 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 38b327ea9be94a8ebe7ee8cb47af9868_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-10T20:28:48,325 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/.tmp/a/3758211768d4443989d577b6de6942cf is 55, key is testCompactedBulkLoadedFiles/a:a/1733862528199/Put/seqid=0 2024-12-10T20:28:48,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741922_1100 (size=6154) 2024-12-10T20:28:48,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741922_1100 (size=6154) 2024-12-10T20:28:48,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741922_1100 (size=6154) 2024-12-10T20:28:48,339 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/.tmp/a/3758211768d4443989d577b6de6942cf as hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/3758211768d4443989d577b6de6942cf 2024-12-10T20:28:48,345 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 6b0091041a95efa38bb4abdd89c08422/a of 6b0091041a95efa38bb4abdd89c08422 into 3758211768d4443989d577b6de6942cf(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-10T20:28:48,345 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 6b0091041a95efa38bb4abdd89c08422: 2024-12-10T20:28:48,345 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-10T20:28:48,345 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-10T20:28:48,375 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33493/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862528161, size=0 (0bytes) 2024-12-10T20:28:48,375 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33493/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862528161 might be still open, length is 0 2024-12-10T20:28:48,375 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33493/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862528161 2024-12-10T20:28:48,376 WARN [IPC Server handler 3 on default port 33493 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862528161 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-10T20:28:48,376 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33493/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862528161 after 1ms 2024-12-10T20:28:49,867 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:47976 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:40165:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47976 dst: /127.0.0.1:40165 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40165 remote=/127.0.0.1:47976]. Total timeout mills is 60000, 58477 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:49,868 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:40606 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:36025:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40606 dst: /127.0.0.1:36025 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:49,868 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:48194 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:45577:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48194 dst: /127.0.0.1:45577 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:49,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741917_1101 (size=1368) 2024-12-10T20:28:49,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741917_1101 (size=1368) 2024-12-10T20:28:52,377 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33493/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862528161 after 4002ms 2024-12-10T20:28:52,383 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862528161: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:52,383 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33493/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862528161 took 4008ms 2024-12-10T20:28:52,385 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33493/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862528161; continuing. 2024-12-10T20:28:52,386 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33493/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862528161 so closing down 2024-12-10T20:28:52,386 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T20:28:52,387 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733862528161.temp 2024-12-10T20:28:52,388 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/recovered.edits/0000000000000000003-wal.1733862528161.temp 2024-12-10T20:28:52,388 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T20:28:52,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741923_1102 (size=635) 2024-12-10T20:28:52,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741923_1102 (size=635) 2024-12-10T20:28:52,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741923_1102 (size=635) 2024-12-10T20:28:52,395 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/recovered.edits/0000000000000000003-wal.1733862528161.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-10T20:28:52,396 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/recovered.edits/0000000000000000003-wal.1733862528161.temp to hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/recovered.edits/0000000000000000008 2024-12-10T20:28:52,396 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:33493/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862528161, size=0, length=0, corrupted=false, cancelled=false 2024-12-10T20:28:52,396 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33493/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862528161, journal: Splitting hdfs://localhost:33493/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862528161, size=0 (0bytes) at 1733862528375Finishing writing output for hdfs://localhost:33493/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862528161 so closing down at 1733862532386 (+4011 ms)Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/recovered.edits/0000000000000000003-wal.1733862528161.temp at 1733862532388 (+2 ms)3 split writer threads finished at 1733862532388Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/recovered.edits/0000000000000000003-wal.1733862528161.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733862532395 (+7 ms)Rename recovered edits hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/recovered.edits/0000000000000000003-wal.1733862528161.temp to hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/recovered.edits/0000000000000000008 at 1733862532396 (+1 ms)Processed 5 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:33493/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862528161, size=0, length=0, corrupted=false, cancelled=false at 1733862532396 2024-12-10T20:28:52,397 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33493/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862528161 to hdfs://localhost:33493/hbase/oldWALs/wal.1733862528161 2024-12-10T20:28:52,398 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/recovered.edits/0000000000000000008 2024-12-10T20:28:52,398 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T20:28:52,399 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:52,415 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862532400, exclude list is [], retry=0 2024-12-10T20:28:52,417 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:52,418 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:52,418 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:52,420 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862532400 2024-12-10T20:28:52,420 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:33825:33825)] 2024-12-10T20:28:52,420 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6b0091041a95efa38bb4abdd89c08422, NAME => 'testCompactedBulkLoadedFiles,,1733862528068.6b0091041a95efa38bb4abdd89c08422.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:52,421 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733862528068.6b0091041a95efa38bb4abdd89c08422.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:52,421 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:52,421 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:52,422 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:52,423 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b0091041a95efa38bb4abdd89c08422 columnFamilyName a 2024-12-10T20:28:52,423 DEBUG [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:52,429 DEBUG [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/3758211768d4443989d577b6de6942cf 2024-12-10T20:28:52,434 DEBUG [StoreFileOpener-6b0091041a95efa38bb4abdd89c08422-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 38b327ea9be94a8ebe7ee8cb47af9868_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-10T20:28:52,434 DEBUG [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/38b327ea9be94a8ebe7ee8cb47af9868_SeqId_4_ 2024-12-10T20:28:52,438 DEBUG [StoreFileOpener-6b0091041a95efa38bb4abdd89c08422-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5d82ac8f545a46799c95f8a1ec91811b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-10T20:28:52,438 DEBUG [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/5d82ac8f545a46799c95f8a1ec91811b_SeqId_4_ 2024-12-10T20:28:52,442 DEBUG [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/68237c2b1cb042eda86ea80680a24011 2024-12-10T20:28:52,446 DEBUG [StoreFileOpener-6b0091041a95efa38bb4abdd89c08422-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for eeda043a3403485eb0ed93168217a78d_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-10T20:28:52,446 DEBUG [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/eeda043a3403485eb0ed93168217a78d_SeqId_4_ 2024-12-10T20:28:52,446 WARN [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/38b327ea9be94a8ebe7ee8cb47af9868_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2215eeca 2024-12-10T20:28:52,446 WARN [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/5d82ac8f545a46799c95f8a1ec91811b_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2215eeca 2024-12-10T20:28:52,446 WARN [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/68237c2b1cb042eda86ea80680a24011 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2215eeca 2024-12-10T20:28:52,446 WARN [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/eeda043a3403485eb0ed93168217a78d_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2215eeca 2024-12-10T20:28:52,446 DEBUG [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/38b327ea9be94a8ebe7ee8cb47af9868_SeqId_4_, hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/5d82ac8f545a46799c95f8a1ec91811b_SeqId_4_, hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/68237c2b1cb042eda86ea80680a24011, hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/eeda043a3403485eb0ed93168217a78d_SeqId_4_] to archive 2024-12-10T20:28:52,447 DEBUG [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-10T20:28:52,449 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/38b327ea9be94a8ebe7ee8cb47af9868_SeqId_4_ to hdfs://localhost:33493/hbase/archive/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/38b327ea9be94a8ebe7ee8cb47af9868_SeqId_4_ 2024-12-10T20:28:52,449 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/5d82ac8f545a46799c95f8a1ec91811b_SeqId_4_ to hdfs://localhost:33493/hbase/archive/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/5d82ac8f545a46799c95f8a1ec91811b_SeqId_4_ 2024-12-10T20:28:52,449 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/eeda043a3403485eb0ed93168217a78d_SeqId_4_ to hdfs://localhost:33493/hbase/archive/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/eeda043a3403485eb0ed93168217a78d_SeqId_4_ 2024-12-10T20:28:52,450 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/68237c2b1cb042eda86ea80680a24011 to hdfs://localhost:33493/hbase/archive/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/a/68237c2b1cb042eda86ea80680a24011 2024-12-10T20:28:52,450 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.HStore(327): Store=6b0091041a95efa38bb4abdd89c08422/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:52,450 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:52,451 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b0091041a95efa38bb4abdd89c08422 columnFamilyName b 2024-12-10T20:28:52,451 DEBUG [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:52,451 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.HStore(327): Store=6b0091041a95efa38bb4abdd89c08422/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:52,451 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:52,452 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6b0091041a95efa38bb4abdd89c08422 columnFamilyName c 2024-12-10T20:28:52,452 DEBUG [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:52,452 INFO [StoreOpener-6b0091041a95efa38bb4abdd89c08422-1 {}] regionserver.HStore(327): Store=6b0091041a95efa38bb4abdd89c08422/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:52,452 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:52,453 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:52,454 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:52,455 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/recovered.edits/0000000000000000008 2024-12-10T20:28:52,457 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:52,458 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 6b0091041a95efa38bb4abdd89c08422 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "6b0091041a95efa38bb4abdd89c08422" family_name: "a" compaction_input: "68237c2b1cb042eda86ea80680a24011" compaction_input: "5d82ac8f545a46799c95f8a1ec91811b_SeqId_4_" compaction_input: "eeda043a3403485eb0ed93168217a78d_SeqId_4_" compaction_input: "38b327ea9be94a8ebe7ee8cb47af9868_SeqId_4_" compaction_output: "3758211768d4443989d577b6de6942cf" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733862528068.6b0091041a95efa38bb4abdd89c08422." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-10T20:28:52,458 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-10T20:28:52,458 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/recovered.edits/0000000000000000008 2024-12-10T20:28:52,459 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/recovered.edits/0000000000000000008 2024-12-10T20:28:52,460 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:52,460 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:52,461 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T20:28:52,462 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6b0091041a95efa38bb4abdd89c08422 2024-12-10T20:28:52,464 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testCompactedBulkLoadedFiles/6b0091041a95efa38bb4abdd89c08422/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-10T20:28:52,466 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6b0091041a95efa38bb4abdd89c08422; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69763494, jitterRate=0.039557069540023804}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T20:28:52,466 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6b0091041a95efa38bb4abdd89c08422: Writing region info on filesystem at 1733862532421Initializing all the Stores at 1733862532422 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862532422Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862532422Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862532422Cleaning up temporary data from old regions at 1733862532461 (+39 ms)Region opened successfully at 1733862532466 (+5 ms) 2024-12-10T20:28:52,469 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6b0091041a95efa38bb4abdd89c08422, disabling compactions & flushes 2024-12-10T20:28:52,469 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733862528068.6b0091041a95efa38bb4abdd89c08422. 2024-12-10T20:28:52,469 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733862528068.6b0091041a95efa38bb4abdd89c08422. 2024-12-10T20:28:52,469 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733862528068.6b0091041a95efa38bb4abdd89c08422. after waiting 0 ms 2024-12-10T20:28:52,469 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733862528068.6b0091041a95efa38bb4abdd89c08422. 2024-12-10T20:28:52,470 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733862528068.6b0091041a95efa38bb4abdd89c08422. 2024-12-10T20:28:52,470 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6b0091041a95efa38bb4abdd89c08422: Waiting for close lock at 1733862532469Disabling compacts and flushes for region at 1733862532469Disabling writes for close at 1733862532469Writing region close event to WAL at 1733862532470 (+1 ms)Closed at 1733862532470 2024-12-10T20:28:52,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741924_1103 (size=93) 2024-12-10T20:28:52,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741924_1103 (size=93) 2024-12-10T20:28:52,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741924_1103 (size=93) 2024-12-10T20:28:52,474 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T20:28:52,474 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733862532400) 2024-12-10T20:28:52,480 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T20:28:52,488 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=450 (was 433) Potentially hanging thread: HFileArchiver-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:33493 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1328911672_22 at /127.0.0.1:59496 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1328911672_22 at /127.0.0.1:34232 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:33493 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1328911672_22 at /127.0.0.1:44412 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=299 (was 299), ProcessCount=11 (was 11), AvailableMemoryMB=5702 (was 5968) 2024-12-10T20:28:52,488 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-12-10T20:28:52,499 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=451, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=299, ProcessCount=11, AvailableMemoryMB=5702 2024-12-10T20:28:52,499 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-12-10T20:28:52,512 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:52,513 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T20:28:52,514 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T20:28:52,516 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-27622291, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/hregion-27622291, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:52,526 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-27622291/hregion-27622291.1733862532516, exclude list is [], retry=0 2024-12-10T20:28:52,529 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:52,529 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:52,529 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:52,530 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-27622291/hregion-27622291.1733862532516 2024-12-10T20:28:52,531 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:46745:46745)] 2024-12-10T20:28:52,531 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e7ec963aa9cb8e38c4d8496edfcea0e8, NAME => 'testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33493/hbase 2024-12-10T20:28:52,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741926_1105 (size=67) 2024-12-10T20:28:52,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741926_1105 (size=67) 2024-12-10T20:28:52,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741926_1105 (size=67) 2024-12-10T20:28:52,538 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:52,540 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,541 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7ec963aa9cb8e38c4d8496edfcea0e8 columnFamilyName a 2024-12-10T20:28:52,541 DEBUG [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:52,541 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(327): Store=e7ec963aa9cb8e38c4d8496edfcea0e8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:52,542 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,543 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7ec963aa9cb8e38c4d8496edfcea0e8 columnFamilyName b 2024-12-10T20:28:52,543 DEBUG [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:52,543 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(327): Store=e7ec963aa9cb8e38c4d8496edfcea0e8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:52,543 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,544 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7ec963aa9cb8e38c4d8496edfcea0e8 columnFamilyName c 2024-12-10T20:28:52,544 DEBUG [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:52,545 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(327): Store=e7ec963aa9cb8e38c4d8496edfcea0e8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:52,545 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,545 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,546 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,547 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,547 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,547 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T20:28:52,548 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,551 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T20:28:52,552 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e7ec963aa9cb8e38c4d8496edfcea0e8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71514701, jitterRate=0.06565208733081818}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T20:28:52,552 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e7ec963aa9cb8e38c4d8496edfcea0e8: Writing region info on filesystem at 1733862532539Initializing all the Stores at 1733862532539Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862532539Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862532539Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862532539Cleaning up temporary data from old regions at 1733862532547 (+8 ms)Region opened successfully at 1733862532552 (+5 ms) 2024-12-10T20:28:52,552 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e7ec963aa9cb8e38c4d8496edfcea0e8, disabling compactions & flushes 2024-12-10T20:28:52,552 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8. 2024-12-10T20:28:52,552 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8. 2024-12-10T20:28:52,552 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8. after waiting 0 ms 2024-12-10T20:28:52,552 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8. 2024-12-10T20:28:52,552 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8. 2024-12-10T20:28:52,553 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e7ec963aa9cb8e38c4d8496edfcea0e8: Waiting for close lock at 1733862532552Disabling compacts and flushes for region at 1733862532552Disabling writes for close at 1733862532552Writing region close event to WAL at 1733862532552Closed at 1733862532552 2024-12-10T20:28:52,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741925_1104 (size=93) 2024-12-10T20:28:52,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741925_1104 (size=93) 2024-12-10T20:28:52,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741925_1104 (size=93) 2024-12-10T20:28:52,557 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T20:28:52,557 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-27622291:(num 1733862532516) 2024-12-10T20:28:52,558 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T20:28:52,559 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:52,570 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532560, exclude list is [], retry=0 2024-12-10T20:28:52,573 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:52,573 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:52,573 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:52,574 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532560 2024-12-10T20:28:52,575 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:40461:40461)] 2024-12-10T20:28:52,575 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e7ec963aa9cb8e38c4d8496edfcea0e8, NAME => 'testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:52,575 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:52,575 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,575 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,576 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,577 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7ec963aa9cb8e38c4d8496edfcea0e8 columnFamilyName a 2024-12-10T20:28:52,577 DEBUG [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:52,578 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(327): Store=e7ec963aa9cb8e38c4d8496edfcea0e8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:52,578 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,578 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7ec963aa9cb8e38c4d8496edfcea0e8 columnFamilyName b 2024-12-10T20:28:52,578 DEBUG [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:52,579 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(327): Store=e7ec963aa9cb8e38c4d8496edfcea0e8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:52,579 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,580 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7ec963aa9cb8e38c4d8496edfcea0e8 columnFamilyName c 2024-12-10T20:28:52,580 DEBUG [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:52,580 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(327): Store=e7ec963aa9cb8e38c4d8496edfcea0e8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:52,580 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,581 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,582 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,583 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,583 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,583 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T20:28:52,585 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,585 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e7ec963aa9cb8e38c4d8496edfcea0e8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65737709, jitterRate=-0.020431801676750183}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T20:28:52,586 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e7ec963aa9cb8e38c4d8496edfcea0e8: Writing region info on filesystem at 1733862532575Initializing all the Stores at 1733862532576 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862532576Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862532576Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862532576Cleaning up temporary data from old regions at 1733862532583 (+7 ms)Region opened successfully at 1733862532586 (+3 ms) 2024-12-10T20:28:52,591 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e7ec963aa9cb8e38c4d8496edfcea0e8 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-10T20:28:52,611 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/.tmp/a/9925c57a15e04917af0e38a2563fea2d is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733862532586/Put/seqid=0 2024-12-10T20:28:52,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741928_1107 (size=5958) 2024-12-10T20:28:52,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741928_1107 (size=5958) 2024-12-10T20:28:52,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741928_1107 (size=5958) 2024-12-10T20:28:52,617 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/.tmp/a/9925c57a15e04917af0e38a2563fea2d 2024-12-10T20:28:52,623 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/.tmp/a/9925c57a15e04917af0e38a2563fea2d as hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/a/9925c57a15e04917af0e38a2563fea2d 2024-12-10T20:28:52,627 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/a/9925c57a15e04917af0e38a2563fea2d, entries=10, sequenceid=13, filesize=5.8 K 2024-12-10T20:28:52,628 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for e7ec963aa9cb8e38c4d8496edfcea0e8 in 37ms, sequenceid=13, compaction requested=false 2024-12-10T20:28:52,628 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e7ec963aa9cb8e38c4d8496edfcea0e8: 2024-12-10T20:28:52,642 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e7ec963aa9cb8e38c4d8496edfcea0e8, disabling compactions & flushes 2024-12-10T20:28:52,642 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8. 2024-12-10T20:28:52,642 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8. 2024-12-10T20:28:52,642 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8. after waiting 0 ms 2024-12-10T20:28:52,642 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8. 2024-12-10T20:28:52,643 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8. 2024-12-10T20:28:52,643 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8. 2024-12-10T20:28:52,643 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e7ec963aa9cb8e38c4d8496edfcea0e8: Waiting for close lock at 1733862532642Disabling compacts and flushes for region at 1733862532642Disabling writes for close at 1733862532642Writing region close event to WAL at 1733862532643 (+1 ms)Closed at 1733862532643 2024-12-10T20:28:52,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741927_1106 (size=2805) 2024-12-10T20:28:52,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741927_1106 (size=2805) 2024-12-10T20:28:52,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741927_1106 (size=2805) 2024-12-10T20:28:52,659 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532560, size=2.7 K (2805bytes) 2024-12-10T20:28:52,659 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532560 2024-12-10T20:28:52,659 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532560 after 0ms 2024-12-10T20:28:52,661 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532560: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:52,661 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532560 took 3ms 2024-12-10T20:28:52,663 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532560 so closing down 2024-12-10T20:28:52,664 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T20:28:52,665 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733862532560.temp 2024-12-10T20:28:52,666 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000003-wal.1733862532560.temp 2024-12-10T20:28:52,666 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T20:28:52,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741929_1108 (size=2312) 2024-12-10T20:28:52,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741929_1108 (size=2312) 2024-12-10T20:28:52,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741929_1108 (size=2312) 2024-12-10T20:28:52,672 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000003-wal.1733862532560.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-10T20:28:52,673 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000003-wal.1733862532560.temp to hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000035 2024-12-10T20:28:52,673 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 11 ms; skipped=2; WAL=hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532560, size=2.7 K, length=2805, corrupted=false, cancelled=false 2024-12-10T20:28:52,674 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532560, journal: Splitting hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532560, size=2.7 K (2805bytes) at 1733862532659Finishing writing output for hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532560 so closing down at 1733862532663 (+4 ms)Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000003-wal.1733862532560.temp at 1733862532666 (+3 ms)3 split writer threads finished at 1733862532666Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000003-wal.1733862532560.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733862532672 (+6 ms)Rename recovered edits hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000003-wal.1733862532560.temp to hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000035 at 1733862532673 (+1 ms)Processed 32 edits across 1 Regions in 11 ms; skipped=2; WAL=hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532560, size=2.7 K, length=2805, corrupted=false, cancelled=false at 1733862532674 (+1 ms) 2024-12-10T20:28:52,675 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532560 to hdfs://localhost:33493/hbase/oldWALs/wal.1733862532560 2024-12-10T20:28:52,676 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000035 2024-12-10T20:28:52,676 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T20:28:52,678 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:52,692 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532678, exclude list is [], retry=0 2024-12-10T20:28:52,694 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:52,694 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:52,695 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:52,696 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532678 2024-12-10T20:28:52,696 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46745:46745),(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:40461:40461)] 2024-12-10T20:28:52,696 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e7ec963aa9cb8e38c4d8496edfcea0e8, NAME => 'testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8.', STARTKEY => '', ENDKEY => ''} 2024-12-10T20:28:52,696 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:52,696 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,696 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,698 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,698 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7ec963aa9cb8e38c4d8496edfcea0e8 columnFamilyName a 2024-12-10T20:28:52,698 DEBUG [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:52,703 DEBUG [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/a/9925c57a15e04917af0e38a2563fea2d 2024-12-10T20:28:52,703 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(327): Store=e7ec963aa9cb8e38c4d8496edfcea0e8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:52,703 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,705 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7ec963aa9cb8e38c4d8496edfcea0e8 columnFamilyName b 2024-12-10T20:28:52,705 DEBUG [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:52,705 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(327): Store=e7ec963aa9cb8e38c4d8496edfcea0e8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:52,705 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,706 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7ec963aa9cb8e38c4d8496edfcea0e8 columnFamilyName c 2024-12-10T20:28:52,706 DEBUG [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:52,706 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(327): Store=e7ec963aa9cb8e38c4d8496edfcea0e8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:52,707 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,707 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,709 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,709 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000035 2024-12-10T20:28:52,711 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:52,712 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000035 2024-12-10T20:28:52,713 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e7ec963aa9cb8e38c4d8496edfcea0e8 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-10T20:28:52,728 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/.tmp/b/316e7937b2e34822aa552f66062e503c is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733862532628/Put/seqid=0 2024-12-10T20:28:52,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741931_1110 (size=5958) 2024-12-10T20:28:52,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741931_1110 (size=5958) 2024-12-10T20:28:52,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741931_1110 (size=5958) 2024-12-10T20:28:52,734 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/.tmp/b/316e7937b2e34822aa552f66062e503c 2024-12-10T20:28:52,751 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/.tmp/c/20d96def9cb548969bd8c0c6d89434d9 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733862532634/Put/seqid=0 2024-12-10T20:28:52,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741932_1111 (size=5958) 2024-12-10T20:28:52,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741932_1111 (size=5958) 2024-12-10T20:28:52,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741932_1111 (size=5958) 2024-12-10T20:28:52,757 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/.tmp/c/20d96def9cb548969bd8c0c6d89434d9 2024-12-10T20:28:52,763 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/.tmp/b/316e7937b2e34822aa552f66062e503c as hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/b/316e7937b2e34822aa552f66062e503c 2024-12-10T20:28:52,767 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/b/316e7937b2e34822aa552f66062e503c, entries=10, sequenceid=35, filesize=5.8 K 2024-12-10T20:28:52,768 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/.tmp/c/20d96def9cb548969bd8c0c6d89434d9 as hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/c/20d96def9cb548969bd8c0c6d89434d9 2024-12-10T20:28:52,773 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/c/20d96def9cb548969bd8c0c6d89434d9, entries=10, sequenceid=35, filesize=5.8 K 2024-12-10T20:28:52,773 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for e7ec963aa9cb8e38c4d8496edfcea0e8 in 61ms, sequenceid=35, compaction requested=false; wal=null 2024-12-10T20:28:52,774 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000035 2024-12-10T20:28:52,774 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,775 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,775 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T20:28:52,776 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:52,778 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-10T20:28:52,779 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e7ec963aa9cb8e38c4d8496edfcea0e8; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66872475, jitterRate=-0.0035224705934524536}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T20:28:52,779 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e7ec963aa9cb8e38c4d8496edfcea0e8: Writing region info on filesystem at 1733862532696Initializing all the Stores at 1733862532697 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862532697Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862532698 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862532698Obtaining lock to block concurrent updates at 1733862532713 (+15 ms)Preparing flush snapshotting stores in e7ec963aa9cb8e38c4d8496edfcea0e8 at 1733862532713Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733862532713Flushing stores of testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8. at 1733862532713Flushing e7ec963aa9cb8e38c4d8496edfcea0e8/b: creating writer at 1733862532713Flushing e7ec963aa9cb8e38c4d8496edfcea0e8/b: appending metadata at 1733862532728 (+15 ms)Flushing e7ec963aa9cb8e38c4d8496edfcea0e8/b: closing flushed file at 1733862532728Flushing e7ec963aa9cb8e38c4d8496edfcea0e8/c: creating writer at 1733862532738 (+10 ms)Flushing e7ec963aa9cb8e38c4d8496edfcea0e8/c: appending metadata at 1733862532751 (+13 ms)Flushing e7ec963aa9cb8e38c4d8496edfcea0e8/c: closing flushed file at 1733862532751Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68a953b7: reopening flushed file at 1733862532762 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cef5540: reopening flushed file at 1733862532768 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for e7ec963aa9cb8e38c4d8496edfcea0e8 in 61ms, sequenceid=35, compaction requested=false; wal=null at 1733862532773 (+5 ms)Cleaning up temporary data from old regions at 1733862532775 (+2 ms)Region opened successfully at 1733862532779 (+4 ms) 2024-12-10T20:28:52,835 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532678, size=0 (0bytes) 2024-12-10T20:28:52,835 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532678 might be still open, length is 0 2024-12-10T20:28:52,835 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532678 2024-12-10T20:28:52,836 WARN [IPC Server handler 1 on default port 33493 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532678 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-10T20:28:52,836 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532678 after 1ms 2024-12-10T20:28:55,722 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:34324 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:40165:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34324 dst: /127.0.0.1:40165 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40165 remote=/127.0.0.1:34324]. Total timeout mills is 60000, 57080 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:55,723 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:44482 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:36025:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44482 dst: /127.0.0.1:36025 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:55,723 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-847481613_22 at /127.0.0.1:59546 [Receiving block BP-76190982-172.17.0.3-1733862492788:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:45577:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59546 dst: /127.0.0.1:45577 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:55,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741930_1112 (size=2304) 2024-12-10T20:28:55,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741930_1112 (size=2304) 2024-12-10T20:28:55,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741930_1112 (size=2304) 2024-12-10T20:28:56,837 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532678 after 4001ms 2024-12-10T20:28:56,840 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532678: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:56,840 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532678 took 4005ms 2024-12-10T20:28:56,850 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532678; continuing. 2024-12-10T20:28:56,850 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532678 so closing down 2024-12-10T20:28:56,851 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-10T20:28:56,852 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733862532678.temp 2024-12-10T20:28:56,861 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000037-wal.1733862532678.temp 2024-12-10T20:28:56,861 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-10T20:28:56,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741933_1113 (size=2312) 2024-12-10T20:28:56,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741933_1113 (size=2312) 2024-12-10T20:28:56,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741933_1113 (size=2312) 2024-12-10T20:28:56,871 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000037-wal.1733862532678.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-10T20:28:56,873 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000037-wal.1733862532678.temp to hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000066 2024-12-10T20:28:56,873 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532678, size=0, length=0, corrupted=false, cancelled=false 2024-12-10T20:28:56,873 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532678, journal: Splitting hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532678, size=0 (0bytes) at 1733862532835Finishing writing output for hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532678 so closing down at 1733862536851 (+4016 ms)Creating recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000037-wal.1733862532678.temp at 1733862536861 (+10 ms)3 split writer threads finished at 1733862536861Closed recovered edits writer path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000037-wal.1733862532678.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733862536871 (+10 ms)Rename recovered edits hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000037-wal.1733862532678.temp to hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000066 at 1733862536873 (+2 ms)Processed 30 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532678, size=0, length=0, corrupted=false, cancelled=false at 1733862536873 2024-12-10T20:28:56,874 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532678 to hdfs://localhost:33493/hbase/oldWALs/wal.1733862532678 2024-12-10T20:28:56,875 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000066 2024-12-10T20:28:56,875 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-10T20:28:56,877 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33493/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511, archiveDir=hdfs://localhost:33493/hbase/oldWALs, maxLogs=32 2024-12-10T20:28:56,892 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862536877, exclude list is [], retry=0 2024-12-10T20:28:56,895 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36025,DS-23f617a0-57ed-45c5-9483-22aefb6c6bfe,DISK] 2024-12-10T20:28:56,895 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45577,DS-64c1773a-d76f-453f-9ced-7a9998f1aa65,DISK] 2024-12-10T20:28:56,896 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40165,DS-ab2a9f30-80ab-4099-9aba-2b96709a16db,DISK] 2024-12-10T20:28:56,897 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862536877 2024-12-10T20:28:56,898 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33825:33825),(127.0.0.1/127.0.0.1:40461:40461),(127.0.0.1/127.0.0.1:46745:46745)] 2024-12-10T20:28:56,898 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T20:28:56,900 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:56,901 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7ec963aa9cb8e38c4d8496edfcea0e8 columnFamilyName a 2024-12-10T20:28:56,901 DEBUG [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:56,907 DEBUG [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/a/9925c57a15e04917af0e38a2563fea2d 2024-12-10T20:28:56,907 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(327): Store=e7ec963aa9cb8e38c4d8496edfcea0e8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:56,907 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:56,908 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7ec963aa9cb8e38c4d8496edfcea0e8 columnFamilyName b 2024-12-10T20:28:56,908 DEBUG [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:56,916 DEBUG [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/b/316e7937b2e34822aa552f66062e503c 2024-12-10T20:28:56,916 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(327): Store=e7ec963aa9cb8e38c4d8496edfcea0e8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:56,916 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:56,917 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7ec963aa9cb8e38c4d8496edfcea0e8 columnFamilyName c 2024-12-10T20:28:56,917 DEBUG [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T20:28:56,924 DEBUG [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/c/20d96def9cb548969bd8c0c6d89434d9 2024-12-10T20:28:56,925 INFO [StoreOpener-e7ec963aa9cb8e38c4d8496edfcea0e8-1 {}] regionserver.HStore(327): Store=e7ec963aa9cb8e38c4d8496edfcea0e8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T20:28:56,925 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:56,925 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:56,927 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:56,927 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000066 2024-12-10T20:28:56,929 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-10T20:28:56,933 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000066 2024-12-10T20:28:56,934 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e7ec963aa9cb8e38c4d8496edfcea0e8 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-10T20:28:56,954 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/.tmp/a/05e4f26f69a74c01b069c42ccd05b653 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733862532784/Put/seqid=0 2024-12-10T20:28:56,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741935_1115 (size=5958) 2024-12-10T20:28:56,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741935_1115 (size=5958) 2024-12-10T20:28:56,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741935_1115 (size=5958) 2024-12-10T20:28:56,970 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/.tmp/a/05e4f26f69a74c01b069c42ccd05b653 2024-12-10T20:28:56,997 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/.tmp/b/164573ffa77f483987a042763b8e0ab2 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733862532791/Put/seqid=0 2024-12-10T20:28:57,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741936_1116 (size=5958) 2024-12-10T20:28:57,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741936_1116 (size=5958) 2024-12-10T20:28:57,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741936_1116 (size=5958) 2024-12-10T20:28:57,008 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/.tmp/b/164573ffa77f483987a042763b8e0ab2 2024-12-10T20:28:57,034 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/.tmp/c/dec460f157034dbbac0e49d3456f76fb is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733862532797/Put/seqid=0 2024-12-10T20:28:57,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741937_1117 (size=5958) 2024-12-10T20:28:57,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741937_1117 (size=5958) 2024-12-10T20:28:57,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741937_1117 (size=5958) 2024-12-10T20:28:57,049 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/.tmp/c/dec460f157034dbbac0e49d3456f76fb 2024-12-10T20:28:57,057 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/.tmp/a/05e4f26f69a74c01b069c42ccd05b653 as hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/a/05e4f26f69a74c01b069c42ccd05b653 2024-12-10T20:28:57,063 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/a/05e4f26f69a74c01b069c42ccd05b653, entries=10, sequenceid=66, filesize=5.8 K 2024-12-10T20:28:57,064 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/.tmp/b/164573ffa77f483987a042763b8e0ab2 as hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/b/164573ffa77f483987a042763b8e0ab2 2024-12-10T20:28:57,072 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/b/164573ffa77f483987a042763b8e0ab2, entries=10, sequenceid=66, filesize=5.8 K 2024-12-10T20:28:57,073 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/.tmp/c/dec460f157034dbbac0e49d3456f76fb as hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/c/dec460f157034dbbac0e49d3456f76fb 2024-12-10T20:28:57,078 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/c/dec460f157034dbbac0e49d3456f76fb, entries=10, sequenceid=66, filesize=5.8 K 2024-12-10T20:28:57,079 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for e7ec963aa9cb8e38c4d8496edfcea0e8 in 146ms, sequenceid=66, compaction requested=false; wal=null 2024-12-10T20:28:57,080 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/0000000000000000066 2024-12-10T20:28:57,081 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:57,081 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:57,083 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-10T20:28:57,085 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e7ec963aa9cb8e38c4d8496edfcea0e8 2024-12-10T20:28:57,088 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/hbase/data/default/testReplayEditsWrittenViaHRegion/e7ec963aa9cb8e38c4d8496edfcea0e8/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-10T20:28:57,089 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e7ec963aa9cb8e38c4d8496edfcea0e8; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73780791, jitterRate=0.09941945970058441}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-10T20:28:57,089 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e7ec963aa9cb8e38c4d8496edfcea0e8: Writing region info on filesystem at 1733862536898Initializing all the Stores at 1733862536899 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862536899Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862536900 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733862536900Obtaining lock to block concurrent updates at 1733862536934 (+34 ms)Preparing flush snapshotting stores in e7ec963aa9cb8e38c4d8496edfcea0e8 at 1733862536934Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733862536934Flushing stores of testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8. at 1733862536934Flushing e7ec963aa9cb8e38c4d8496edfcea0e8/a: creating writer at 1733862536934Flushing e7ec963aa9cb8e38c4d8496edfcea0e8/a: appending metadata at 1733862536953 (+19 ms)Flushing e7ec963aa9cb8e38c4d8496edfcea0e8/a: closing flushed file at 1733862536954 (+1 ms)Flushing e7ec963aa9cb8e38c4d8496edfcea0e8/b: creating writer at 1733862536977 (+23 ms)Flushing e7ec963aa9cb8e38c4d8496edfcea0e8/b: appending metadata at 1733862536996 (+19 ms)Flushing e7ec963aa9cb8e38c4d8496edfcea0e8/b: closing flushed file at 1733862536996Flushing e7ec963aa9cb8e38c4d8496edfcea0e8/c: creating writer at 1733862537014 (+18 ms)Flushing e7ec963aa9cb8e38c4d8496edfcea0e8/c: appending metadata at 1733862537033 (+19 ms)Flushing e7ec963aa9cb8e38c4d8496edfcea0e8/c: closing flushed file at 1733862537034 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ffba5e6: reopening flushed file at 1733862537056 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@652b9025: reopening flushed file at 1733862537063 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36c0125d: reopening flushed file at 1733862537072 (+9 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for e7ec963aa9cb8e38c4d8496edfcea0e8 in 146ms, sequenceid=66, compaction requested=false; wal=null at 1733862537079 (+7 ms)Cleaning up temporary data from old regions at 1733862537081 (+2 ms)Region opened successfully at 1733862537089 (+8 ms) 2024-12-10T20:28:57,119 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e7ec963aa9cb8e38c4d8496edfcea0e8, disabling compactions & flushes 2024-12-10T20:28:57,119 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8. 2024-12-10T20:28:57,119 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8. 2024-12-10T20:28:57,119 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8. after waiting 0 ms 2024-12-10T20:28:57,119 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8. 2024-12-10T20:28:57,127 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733862532512.e7ec963aa9cb8e38c4d8496edfcea0e8. 2024-12-10T20:28:57,127 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e7ec963aa9cb8e38c4d8496edfcea0e8: Waiting for close lock at 1733862537119Disabling compacts and flushes for region at 1733862537119Disabling writes for close at 1733862537119Writing region close event to WAL at 1733862537127 (+8 ms)Closed at 1733862537127 2024-12-10T20:28:57,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741934_1114 (size=93) 2024-12-10T20:28:57,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741934_1114 (size=93) 2024-12-10T20:28:57,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741934_1114 (size=93) 2024-12-10T20:28:57,134 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-10T20:28:57,134 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733862536877) 2024-12-10T20:28:57,148 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=457 (was 451) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:33493 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37367 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:33493 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:37367 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1785583234_22 at /127.0.0.1:34364 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1785583234_22 at /127.0.0.1:59610 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1785583234_22 at /127.0.0.1:44522 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1416 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=339 (was 299) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6069 (was 5702) - AvailableMemoryMB LEAK? - 2024-12-10T20:28:57,148 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1416 is superior to 1024 2024-12-10T20:28:57,148 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-10T20:28:57,148 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T20:28:57,148 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T20:28:57,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:28:57,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:28:57,149 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T20:28:57,149 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-10T20:28:57,150 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=750567030, stopped=false 2024-12-10T20:28:57,150 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=03a2ae042c95,34471,1733862497789 2024-12-10T20:28:57,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T20:28:57,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34629-0x100116a21c40001, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T20:28:57,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:57,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41599-0x100116a21c40002, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T20:28:57,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41599-0x100116a21c40002, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:57,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34629-0x100116a21c40001, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:57,215 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T20:28:57,215 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T20:28:57,215 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T20:28:57,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:28:57,216 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '03a2ae042c95,34629,1733862498540' ***** 2024-12-10T20:28:57,216 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T20:28:57,216 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '03a2ae042c95,41599,1733862498634' ***** 2024-12-10T20:28:57,216 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T20:28:57,216 INFO [RS:0;03a2ae042c95:34629 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T20:28:57,216 INFO [RS:0;03a2ae042c95:34629 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T20:28:57,216 INFO [RS:0;03a2ae042c95:34629 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T20:28:57,216 INFO [RS:0;03a2ae042c95:34629 {}] regionserver.HRegionServer(959): stopping server 03a2ae042c95,34629,1733862498540 2024-12-10T20:28:57,216 INFO [RS:0;03a2ae042c95:34629 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T20:28:57,216 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T20:28:57,216 INFO [RS:0;03a2ae042c95:34629 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;03a2ae042c95:34629. 2024-12-10T20:28:57,216 DEBUG [RS:0;03a2ae042c95:34629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T20:28:57,216 DEBUG [RS:0;03a2ae042c95:34629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:28:57,216 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34629-0x100116a21c40001, quorum=127.0.0.1:53600, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T20:28:57,217 INFO [RS:0;03a2ae042c95:34629 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T20:28:57,217 INFO [RS:0;03a2ae042c95:34629 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T20:28:57,217 INFO [RS:0;03a2ae042c95:34629 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T20:28:57,217 INFO [RS:0;03a2ae042c95:34629 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-10T20:28:57,217 INFO [RS:1;03a2ae042c95:41599 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T20:28:57,217 INFO [RS:1;03a2ae042c95:41599 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T20:28:57,217 INFO [RS:1;03a2ae042c95:41599 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T20:28:57,217 INFO [RS:1;03a2ae042c95:41599 {}] regionserver.HRegionServer(3091): Received CLOSE for 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:57,217 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T20:28:57,218 INFO [RS:1;03a2ae042c95:41599 {}] regionserver.HRegionServer(959): stopping server 03a2ae042c95,41599,1733862498634 2024-12-10T20:28:57,218 INFO [RS:1;03a2ae042c95:41599 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T20:28:57,218 INFO [RS:1;03a2ae042c95:41599 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;03a2ae042c95:41599. 2024-12-10T20:28:57,218 DEBUG [RS:1;03a2ae042c95:41599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T20:28:57,218 DEBUG [RS:1;03a2ae042c95:41599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:28:57,218 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41599-0x100116a21c40002, quorum=127.0.0.1:53600, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T20:28:57,218 INFO [RS:1;03a2ae042c95:41599 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-10T20:28:57,218 DEBUG [RS:1;03a2ae042c95:41599 {}] regionserver.HRegionServer(1325): Online Regions={6801f39a7c6a3dc7b05c32d2cd7c10e1=testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1.} 2024-12-10T20:28:57,218 DEBUG [RS:1;03a2ae042c95:41599 {}] regionserver.HRegionServer(1351): Waiting on 6801f39a7c6a3dc7b05c32d2cd7c10e1 2024-12-10T20:28:57,218 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T20:28:57,218 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6801f39a7c6a3dc7b05c32d2cd7c10e1, disabling compactions & flushes 2024-12-10T20:28:57,219 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:57,219 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:57,219 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. after waiting 0 ms 2024-12-10T20:28:57,219 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:57,219 INFO [RS:0;03a2ae042c95:34629 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-10T20:28:57,219 DEBUG [RS:0;03a2ae042c95:34629 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-10T20:28:57,219 DEBUG [RS:0;03a2ae042c95:34629 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T20:28:57,219 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T20:28:57,219 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T20:28:57,219 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T20:28:57,219 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T20:28:57,219 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T20:28:57,219 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-10T20:28:57,225 INFO [regionserver/03a2ae042c95:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-10T20:28:57,225 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/default/testReplayEditsAfterRegionMovedWithMultiCF/6801f39a7c6a3dc7b05c32d2cd7c10e1/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-10T20:28:57,225 INFO [regionserver/03a2ae042c95:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-10T20:28:57,225 INFO [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:57,226 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6801f39a7c6a3dc7b05c32d2cd7c10e1: Waiting for close lock at 1733862537218Running coprocessor pre-close hooks at 1733862537218Disabling compacts and flushes for region at 1733862537218Disabling writes for close at 1733862537219 (+1 ms)Writing region close event to WAL at 1733862537221 (+2 ms)Running coprocessor post-close hooks at 1733862537225 (+4 ms)Closed at 1733862537225 2024-12-10T20:28:57,226 DEBUG [RS_CLOSE_REGION-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1. 2024-12-10T20:28:57,239 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740/.tmp/info/93f56c7e67ac4c03af664d6cbe31bee3 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733862513655.6801f39a7c6a3dc7b05c32d2cd7c10e1./info:regioninfo/1733862516798/Put/seqid=0 2024-12-10T20:28:57,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741938_1118 (size=8243) 2024-12-10T20:28:57,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741938_1118 (size=8243) 2024-12-10T20:28:57,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741938_1118 (size=8243) 2024-12-10T20:28:57,248 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740/.tmp/info/93f56c7e67ac4c03af664d6cbe31bee3 2024-12-10T20:28:57,262 INFO [regionserver/03a2ae042c95:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T20:28:57,262 INFO [regionserver/03a2ae042c95:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T20:28:57,263 INFO [regionserver/03a2ae042c95:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-10T20:28:57,263 INFO [regionserver/03a2ae042c95:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-10T20:28:57,274 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740/.tmp/ns/5736b0ca5a43444a9323c43cc1a5c05b is 43, key is default/ns:d/1733862501073/Put/seqid=0 2024-12-10T20:28:57,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741939_1119 (size=5153) 2024-12-10T20:28:57,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741939_1119 (size=5153) 2024-12-10T20:28:57,283 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740/.tmp/ns/5736b0ca5a43444a9323c43cc1a5c05b 2024-12-10T20:28:57,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741939_1119 (size=5153) 2024-12-10T20:28:57,305 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740/.tmp/table/4e7fcbeabc00478a9fd5cdcdf06cf426 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733862514082/Put/seqid=0 2024-12-10T20:28:57,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741940_1120 (size=5431) 2024-12-10T20:28:57,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741940_1120 (size=5431) 2024-12-10T20:28:57,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741940_1120 (size=5431) 2024-12-10T20:28:57,317 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740/.tmp/table/4e7fcbeabc00478a9fd5cdcdf06cf426 2024-12-10T20:28:57,327 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740/.tmp/info/93f56c7e67ac4c03af664d6cbe31bee3 as hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740/info/93f56c7e67ac4c03af664d6cbe31bee3 2024-12-10T20:28:57,333 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740/info/93f56c7e67ac4c03af664d6cbe31bee3, entries=18, sequenceid=21, filesize=8.0 K 2024-12-10T20:28:57,334 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740/.tmp/ns/5736b0ca5a43444a9323c43cc1a5c05b as hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740/ns/5736b0ca5a43444a9323c43cc1a5c05b 2024-12-10T20:28:57,340 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740/ns/5736b0ca5a43444a9323c43cc1a5c05b, entries=2, sequenceid=21, filesize=5.0 K 2024-12-10T20:28:57,341 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740/.tmp/table/4e7fcbeabc00478a9fd5cdcdf06cf426 as hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740/table/4e7fcbeabc00478a9fd5cdcdf06cf426 2024-12-10T20:28:57,347 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740/table/4e7fcbeabc00478a9fd5cdcdf06cf426, entries=2, sequenceid=21, filesize=5.3 K 2024-12-10T20:28:57,349 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 130ms, sequenceid=21, compaction requested=false 2024-12-10T20:28:57,354 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-10T20:28:57,354 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T20:28:57,354 INFO [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T20:28:57,354 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733862537219Running coprocessor pre-close hooks at 1733862537219Disabling compacts and flushes for region at 1733862537219Disabling writes for close at 1733862537219Obtaining lock to block concurrent updates at 1733862537219Preparing flush snapshotting stores in 1588230740 at 1733862537219Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733862537220 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733862537220Flushing 1588230740/info: creating writer at 1733862537221 (+1 ms)Flushing 1588230740/info: appending metadata at 1733862537238 (+17 ms)Flushing 1588230740/info: closing flushed file at 1733862537238Flushing 1588230740/ns: creating writer at 1733862537254 (+16 ms)Flushing 1588230740/ns: appending metadata at 1733862537273 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1733862537273Flushing 1588230740/table: creating writer at 1733862537288 (+15 ms)Flushing 1588230740/table: appending metadata at 1733862537305 (+17 ms)Flushing 1588230740/table: closing flushed file at 1733862537305Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e23b6be: reopening flushed file at 1733862537326 (+21 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d851b7c: reopening flushed file at 1733862537334 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39f83dd4: reopening flushed file at 1733862537340 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 130ms, sequenceid=21, compaction requested=false at 1733862537349 (+9 ms)Writing region close event to WAL at 1733862537350 (+1 ms)Running coprocessor post-close hooks at 1733862537354 (+4 ms)Closed at 1733862537354 2024-12-10T20:28:57,355 DEBUG [RS_CLOSE_META-regionserver/03a2ae042c95:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T20:28:57,418 INFO [RS:1;03a2ae042c95:41599 {}] regionserver.HRegionServer(976): stopping server 03a2ae042c95,41599,1733862498634; all regions closed. 2024-12-10T20:28:57,419 INFO [RS:0;03a2ae042c95:34629 {}] regionserver.HRegionServer(976): stopping server 03a2ae042c95,34629,1733862498540; all regions closed. 2024-12-10T20:28:57,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741834_1010 (size=841) 2024-12-10T20:28:57,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741834_1010 (size=841) 2024-12-10T20:28:57,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741834_1010 (size=841) 2024-12-10T20:28:57,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741836_1012 (size=3783) 2024-12-10T20:28:57,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741836_1012 (size=3783) 2024-12-10T20:28:57,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741836_1012 (size=3783) 2024-12-10T20:28:57,426 DEBUG [RS:1;03a2ae042c95:41599 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/oldWALs 2024-12-10T20:28:57,426 INFO [RS:1;03a2ae042c95:41599 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 03a2ae042c95%2C41599%2C1733862498634:(num 1733862500392) 2024-12-10T20:28:57,426 DEBUG [RS:1;03a2ae042c95:41599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:28:57,426 INFO [RS:1;03a2ae042c95:41599 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T20:28:57,426 INFO [RS:1;03a2ae042c95:41599 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T20:28:57,426 INFO [RS:1;03a2ae042c95:41599 {}] hbase.ChoreService(370): Chore service for: regionserver/03a2ae042c95:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T20:28:57,426 INFO [RS:1;03a2ae042c95:41599 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T20:28:57,426 INFO [RS:1;03a2ae042c95:41599 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T20:28:57,426 INFO [RS:1;03a2ae042c95:41599 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T20:28:57,426 INFO [regionserver/03a2ae042c95:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T20:28:57,426 INFO [RS:1;03a2ae042c95:41599 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T20:28:57,427 INFO [RS:1;03a2ae042c95:41599 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41599 2024-12-10T20:28:57,427 DEBUG [RS:0;03a2ae042c95:34629 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/oldWALs 2024-12-10T20:28:57,427 INFO [RS:0;03a2ae042c95:34629 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 03a2ae042c95%2C34629%2C1733862498540.meta:.meta(num 1733862500878) 2024-12-10T20:28:57,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741835_1011 (size=2054) 2024-12-10T20:28:57,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741835_1011 (size=2054) 2024-12-10T20:28:57,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741835_1011 (size=2054) 2024-12-10T20:28:57,432 DEBUG [RS:0;03a2ae042c95:34629 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/oldWALs 2024-12-10T20:28:57,432 INFO [RS:0;03a2ae042c95:34629 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 03a2ae042c95%2C34629%2C1733862498540:(num 1733862500392) 2024-12-10T20:28:57,432 DEBUG [RS:0;03a2ae042c95:34629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T20:28:57,432 INFO [RS:0;03a2ae042c95:34629 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T20:28:57,432 INFO [RS:0;03a2ae042c95:34629 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T20:28:57,432 INFO [RS:0;03a2ae042c95:34629 {}] hbase.ChoreService(370): Chore service for: regionserver/03a2ae042c95:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T20:28:57,432 INFO [RS:0;03a2ae042c95:34629 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T20:28:57,432 INFO [regionserver/03a2ae042c95:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T20:28:57,432 INFO [RS:0;03a2ae042c95:34629 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34629 2024-12-10T20:28:57,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T20:28:57,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41599-0x100116a21c40002, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/03a2ae042c95,41599,1733862498634 2024-12-10T20:28:57,436 INFO [RS:1;03a2ae042c95:41599 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T20:28:57,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34629-0x100116a21c40001, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/03a2ae042c95,34629,1733862498540 2024-12-10T20:28:57,446 INFO [RS:0;03a2ae042c95:34629 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T20:28:57,457 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [03a2ae042c95,34629,1733862498540] 2024-12-10T20:28:57,477 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/03a2ae042c95,34629,1733862498540 already deleted, retry=false 2024-12-10T20:28:57,478 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 03a2ae042c95,34629,1733862498540 expired; onlineServers=1 2024-12-10T20:28:57,478 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [03a2ae042c95,41599,1733862498634] 2024-12-10T20:28:57,488 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/03a2ae042c95,41599,1733862498634 already deleted, retry=false 2024-12-10T20:28:57,488 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 03a2ae042c95,41599,1733862498634 expired; onlineServers=0 2024-12-10T20:28:57,488 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '03a2ae042c95,34471,1733862497789' ***** 2024-12-10T20:28:57,488 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-10T20:28:57,489 INFO [M:0;03a2ae042c95:34471 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T20:28:57,489 INFO [M:0;03a2ae042c95:34471 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T20:28:57,489 DEBUG [M:0;03a2ae042c95:34471 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-10T20:28:57,489 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-10T20:28:57,489 DEBUG [M:0;03a2ae042c95:34471 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-10T20:28:57,489 DEBUG [master/03a2ae042c95:0:becomeActiveMaster-HFileCleaner.large.0-1733862500010 {}] cleaner.HFileCleaner(306): Exit Thread[master/03a2ae042c95:0:becomeActiveMaster-HFileCleaner.large.0-1733862500010,5,FailOnTimeoutGroup] 2024-12-10T20:28:57,489 DEBUG [master/03a2ae042c95:0:becomeActiveMaster-HFileCleaner.small.0-1733862500011 {}] cleaner.HFileCleaner(306): Exit Thread[master/03a2ae042c95:0:becomeActiveMaster-HFileCleaner.small.0-1733862500011,5,FailOnTimeoutGroup] 2024-12-10T20:28:57,489 INFO [M:0;03a2ae042c95:34471 {}] hbase.ChoreService(370): Chore service for: master/03a2ae042c95:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-10T20:28:57,489 INFO [M:0;03a2ae042c95:34471 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T20:28:57,489 DEBUG [M:0;03a2ae042c95:34471 {}] master.HMaster(1795): Stopping service threads 2024-12-10T20:28:57,490 INFO [M:0;03a2ae042c95:34471 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-10T20:28:57,490 INFO [M:0;03a2ae042c95:34471 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T20:28:57,490 INFO [M:0;03a2ae042c95:34471 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-10T20:28:57,490 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-10T20:28:57,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-10T20:28:57,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T20:28:57,500 DEBUG [M:0;03a2ae042c95:34471 {}] zookeeper.ZKUtil(347): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-10T20:28:57,500 WARN [M:0;03a2ae042c95:34471 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-10T20:28:57,501 INFO [M:0;03a2ae042c95:34471 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/.lastflushedseqids 2024-12-10T20:28:57,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741941_1121 (size=138) 2024-12-10T20:28:57,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741941_1121 (size=138) 2024-12-10T20:28:57,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741941_1121 (size=138) 2024-12-10T20:28:57,515 INFO [M:0;03a2ae042c95:34471 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-10T20:28:57,515 INFO [M:0;03a2ae042c95:34471 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-10T20:28:57,515 DEBUG [M:0;03a2ae042c95:34471 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T20:28:57,515 INFO [M:0;03a2ae042c95:34471 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T20:28:57,515 DEBUG [M:0;03a2ae042c95:34471 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T20:28:57,515 DEBUG [M:0;03a2ae042c95:34471 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T20:28:57,515 DEBUG [M:0;03a2ae042c95:34471 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T20:28:57,515 INFO [M:0;03a2ae042c95:34471 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-12-10T20:28:57,535 DEBUG [M:0;03a2ae042c95:34471 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8aba16ab1e8342dba40287cdf03aab8b is 82, key is hbase:meta,,1/info:regioninfo/1733862500958/Put/seqid=0 2024-12-10T20:28:57,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741942_1122 (size=5672) 2024-12-10T20:28:57,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741942_1122 (size=5672) 2024-12-10T20:28:57,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741942_1122 (size=5672) 2024-12-10T20:28:57,545 INFO [M:0;03a2ae042c95:34471 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8aba16ab1e8342dba40287cdf03aab8b 2024-12-10T20:28:57,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41599-0x100116a21c40002, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T20:28:57,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41599-0x100116a21c40002, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T20:28:57,557 INFO [RS:1;03a2ae042c95:41599 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T20:28:57,557 INFO [RS:1;03a2ae042c95:41599 {}] regionserver.HRegionServer(1031): Exiting; stopping=03a2ae042c95,41599,1733862498634; zookeeper connection closed. 2024-12-10T20:28:57,557 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@11cf80a6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@11cf80a6 2024-12-10T20:28:57,565 DEBUG [M:0;03a2ae042c95:34471 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ea50ee1c00ec4abbadef7cb46c1ae74f is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733862514089/Put/seqid=0 2024-12-10T20:28:57,567 INFO [RS:0;03a2ae042c95:34629 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T20:28:57,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34629-0x100116a21c40001, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T20:28:57,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34629-0x100116a21c40001, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T20:28:57,567 INFO [RS:0;03a2ae042c95:34629 {}] regionserver.HRegionServer(1031): Exiting; stopping=03a2ae042c95,34629,1733862498540; zookeeper connection closed. 2024-12-10T20:28:57,569 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@53ddf9bf {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@53ddf9bf 2024-12-10T20:28:57,569 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-10T20:28:57,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741943_1123 (size=7755) 2024-12-10T20:28:57,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741943_1123 (size=7755) 2024-12-10T20:28:57,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741943_1123 (size=7755) 2024-12-10T20:28:57,582 INFO [M:0;03a2ae042c95:34471 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ea50ee1c00ec4abbadef7cb46c1ae74f 2024-12-10T20:28:57,587 INFO [M:0;03a2ae042c95:34471 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ea50ee1c00ec4abbadef7cb46c1ae74f 2024-12-10T20:28:57,604 DEBUG [M:0;03a2ae042c95:34471 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/62f466ed2255486ebe8dfca67687f57e is 69, key is 03a2ae042c95,34629,1733862498540/rs:state/1733862500068/Put/seqid=0 2024-12-10T20:28:57,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741944_1124 (size=5445) 2024-12-10T20:28:57,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741944_1124 (size=5445) 2024-12-10T20:28:57,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741944_1124 (size=5445) 2024-12-10T20:28:57,622 INFO [M:0;03a2ae042c95:34471 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/62f466ed2255486ebe8dfca67687f57e 2024-12-10T20:28:57,627 INFO [M:0;03a2ae042c95:34471 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 62f466ed2255486ebe8dfca67687f57e 2024-12-10T20:28:57,628 DEBUG [M:0;03a2ae042c95:34471 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8aba16ab1e8342dba40287cdf03aab8b as hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8aba16ab1e8342dba40287cdf03aab8b 2024-12-10T20:28:57,633 INFO [M:0;03a2ae042c95:34471 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8aba16ab1e8342dba40287cdf03aab8b, entries=8, sequenceid=168, filesize=5.5 K 2024-12-10T20:28:57,634 DEBUG [M:0;03a2ae042c95:34471 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ea50ee1c00ec4abbadef7cb46c1ae74f as hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ea50ee1c00ec4abbadef7cb46c1ae74f 2024-12-10T20:28:57,640 INFO [M:0;03a2ae042c95:34471 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ea50ee1c00ec4abbadef7cb46c1ae74f 2024-12-10T20:28:57,640 INFO [M:0;03a2ae042c95:34471 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ea50ee1c00ec4abbadef7cb46c1ae74f, entries=17, sequenceid=168, filesize=7.6 K 2024-12-10T20:28:57,642 DEBUG [M:0;03a2ae042c95:34471 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/62f466ed2255486ebe8dfca67687f57e as hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/62f466ed2255486ebe8dfca67687f57e 2024-12-10T20:28:57,647 INFO [M:0;03a2ae042c95:34471 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 62f466ed2255486ebe8dfca67687f57e 2024-12-10T20:28:57,647 INFO [M:0;03a2ae042c95:34471 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33493/user/jenkins/test-data/48e8a5c5-4af9-1746-7c04-258bedb33c2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/62f466ed2255486ebe8dfca67687f57e, entries=3, sequenceid=168, filesize=5.3 K 2024-12-10T20:28:57,648 INFO [M:0;03a2ae042c95:34471 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 133ms, sequenceid=168, compaction requested=false 2024-12-10T20:28:57,649 INFO [M:0;03a2ae042c95:34471 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T20:28:57,649 DEBUG [M:0;03a2ae042c95:34471 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733862537515Disabling compacts and flushes for region at 1733862537515Disabling writes for close at 1733862537515Obtaining lock to block concurrent updates at 1733862537515Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733862537515Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1733862537516 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733862537516Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733862537516Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733862537535 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733862537535Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733862537550 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733862537565 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733862537565Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733862537587 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733862537604 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733862537604Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@477c639b: reopening flushed file at 1733862537627 (+23 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f55b3d8: reopening flushed file at 1733862537633 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@210c604: reopening flushed file at 1733862537641 (+8 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 133ms, sequenceid=168, compaction requested=false at 1733862537648 (+7 ms)Writing region close event to WAL at 1733862537649 (+1 ms)Closed at 1733862537649 2024-12-10T20:28:57,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45577 is added to blk_1073741830_1006 (size=71337) 2024-12-10T20:28:57,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36025 is added to blk_1073741830_1006 (size=71337) 2024-12-10T20:28:57,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40165 is added to blk_1073741830_1006 (size=71337) 2024-12-10T20:28:57,653 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T20:28:57,653 INFO [M:0;03a2ae042c95:34471 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-10T20:28:57,653 INFO [M:0;03a2ae042c95:34471 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34471 2024-12-10T20:28:57,654 INFO [M:0;03a2ae042c95:34471 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T20:28:57,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T20:28:57,767 INFO [M:0;03a2ae042c95:34471 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T20:28:57,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34471-0x100116a21c40000, quorum=127.0.0.1:53600, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T20:28:57,775 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155/wal.1733862517625 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155/wal.1733862517625 (inode 16655) Holder DFSClient_NONMAPREDUCE_-847481613_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733862517155/wal.1733862517625 (inode 16655) Holder DFSClient_NONMAPREDUCE_-847481613_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-10T20:28:57,777 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733862509295/wal.1733862509389 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:57,777 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733862532511/wal.1733862532678 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:57,780 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766/wal.1733862527000 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766/wal.1733862527000 (inode 16678) Holder DFSClient_NONMAPREDUCE_-847481613_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733862517766/wal.1733862527000 (inode 16678) Holder DFSClient_NONMAPREDUCE_-847481613_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-10T20:28:57,780 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733862502676/wal.1733862502766 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:57,783 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168/wal.1733862527818 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168/wal.1733862527818 (inode 16704) Holder DFSClient_NONMAPREDUCE_-847481613_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733862527168/wal.1733862527818 (inode 16704) Holder DFSClient_NONMAPREDUCE_-847481613_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-10T20:28:57,783 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733862528067/wal.1733862528161 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T20:28:57,785 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal.1733862502545 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal.1733862502545 (inode 16485) Holder DFSClient_NONMAPREDUCE_-847481613_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733862501946/wal.1733862502545 (inode 16485) Holder DFSClient_NONMAPREDUCE_-847481613_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-10T20:28:57,787 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal.1733862501662 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal.1733862501662 (inode 16462) Holder DFSClient_NONMAPREDUCE_-847481613_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733862501358/wal.1733862501662 (inode 16462) Holder DFSClient_NONMAPREDUCE_-847481613_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-10T20:28:57,794 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@35f1150e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T20:28:57,796 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T20:28:57,796 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T20:28:57,796 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T20:28:57,796 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/hadoop.log.dir/,STOPPED} 2024-12-10T20:28:57,799 WARN [BP-76190982-172.17.0.3-1733862492788 heartbeating to localhost/127.0.0.1:33493 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-10T20:28:57,799 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-10T20:28:57,800 WARN [BP-76190982-172.17.0.3-1733862492788 heartbeating to localhost/127.0.0.1:33493 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-76190982-172.17.0.3-1733862492788 (Datanode Uuid 60303f5d-7098-4533-bb59-60644eb83420) service to localhost/127.0.0.1:33493 2024-12-10T20:28:57,800 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-10T20:28:57,801 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/data/data5/current/BP-76190982-172.17.0.3-1733862492788 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T20:28:57,801 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/data/data6/current/BP-76190982-172.17.0.3-1733862492788 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T20:28:57,801 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-10T20:28:57,810 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7bd427b8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T20:28:57,810 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T20:28:57,810 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T20:28:57,810 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T20:28:57,811 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/hadoop.log.dir/,STOPPED} 2024-12-10T20:28:57,812 WARN [BP-76190982-172.17.0.3-1733862492788 heartbeating to localhost/127.0.0.1:33493 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-10T20:28:57,812 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-10T20:28:57,812 WARN [BP-76190982-172.17.0.3-1733862492788 heartbeating to localhost/127.0.0.1:33493 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-76190982-172.17.0.3-1733862492788 (Datanode Uuid 06602c36-db2d-42b9-8c1f-9aba903bd8fd) service to localhost/127.0.0.1:33493 2024-12-10T20:28:57,812 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-10T20:28:57,812 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/data/data3/current/BP-76190982-172.17.0.3-1733862492788 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T20:28:57,813 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/data/data4/current/BP-76190982-172.17.0.3-1733862492788 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T20:28:57,813 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-10T20:28:57,820 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@330740de{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-10T20:28:57,820 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T20:28:57,820 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T20:28:57,820 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T20:28:57,820 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/hadoop.log.dir/,STOPPED} 2024-12-10T20:28:57,821 WARN [BP-76190982-172.17.0.3-1733862492788 heartbeating to localhost/127.0.0.1:33493 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-10T20:28:57,821 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-10T20:28:57,821 WARN [BP-76190982-172.17.0.3-1733862492788 heartbeating to localhost/127.0.0.1:33493 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-76190982-172.17.0.3-1733862492788 (Datanode Uuid 16b431a4-85e7-433c-add7-0b3e71221b62) service to localhost/127.0.0.1:33493 2024-12-10T20:28:57,821 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-10T20:28:57,822 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/data/data1/current/BP-76190982-172.17.0.3-1733862492788 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T20:28:57,822 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/cluster_184ce634-0b3d-b719-d500-2f304ea5cf07/data/data2/current/BP-76190982-172.17.0.3-1733862492788 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-10T20:28:57,822 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-10T20:28:57,830 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3717288f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-10T20:28:57,830 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-10T20:28:57,830 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-10T20:28:57,830 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-10T20:28:57,831 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b776d1d0-2ac8-8867-1679-11db2ad93f19/hadoop.log.dir/,STOPPED} 2024-12-10T20:28:57,841 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-10T20:28:57,899 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down