2024-11-13 00:28:32,897 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-13 00:28:32,911 main DEBUG Took 0.011557 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 00:28:32,911 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 00:28:32,912 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 00:28:32,913 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 00:28:32,914 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:28:32,922 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 00:28:32,936 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:28:32,938 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:28:32,938 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:28:32,938 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:28:32,939 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:28:32,939 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:28:32,940 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:28:32,940 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:28:32,940 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:28:32,941 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:28:32,941 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:28:32,942 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:28:32,942 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:28:32,942 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:28:32,943 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:28:32,943 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:28:32,944 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:28:32,944 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:28:32,944 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:28:32,944 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:28:32,945 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:28:32,945 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:28:32,945 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:28:32,946 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 00:28:32,946 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:28:32,946 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 00:28:32,948 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 00:28:32,949 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 00:28:32,950 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 00:28:32,951 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 00:28:32,952 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 00:28:32,952 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 00:28:32,960 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 00:28:32,962 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 00:28:32,963 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 00:28:32,964 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 00:28:32,964 main DEBUG createAppenders(={Console}) 2024-11-13 00:28:32,965 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-11-13 00:28:32,965 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-13 00:28:32,965 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-11-13 00:28:32,966 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 00:28:32,966 main DEBUG OutputStream closed 2024-11-13 00:28:32,966 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 00:28:32,966 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 00:28:32,967 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-11-13 00:28:33,030 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 00:28:33,032 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 00:28:33,033 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 00:28:33,034 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 00:28:33,034 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 00:28:33,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 00:28:33,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 00:28:33,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 00:28:33,035 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 00:28:33,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 00:28:33,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 00:28:33,036 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 00:28:33,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 00:28:33,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 00:28:33,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 00:28:33,037 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 00:28:33,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 00:28:33,038 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 00:28:33,041 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 00:28:33,041 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-11-13 00:28:33,041 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 00:28:33,042 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-11-13T00:28:33,312 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d 2024-11-13 00:28:33,314 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 00:28:33,315 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T00:28:33,324 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-13T00:28:33,349 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T00:28:33,401 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T00:28:33,401 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T00:28:33,413 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T00:28:33,428 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b, deleteOnExit=true 2024-11-13T00:28:33,428 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T00:28:33,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/test.cache.data in system properties and HBase conf 2024-11-13T00:28:33,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T00:28:33,430 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/hadoop.log.dir in system properties and HBase conf 2024-11-13T00:28:33,430 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T00:28:33,431 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T00:28:33,431 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T00:28:33,521 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T00:28:33,618 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T00:28:33,623 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T00:28:33,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T00:28:33,625 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T00:28:33,625 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T00:28:33,626 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T00:28:33,627 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T00:28:33,627 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T00:28:33,628 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T00:28:33,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T00:28:33,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/nfs.dump.dir in system properties and HBase conf 2024-11-13T00:28:33,630 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/java.io.tmpdir in system properties and HBase conf 2024-11-13T00:28:33,630 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T00:28:33,631 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T00:28:33,631 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T00:28:34,475 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T00:28:34,557 INFO [Time-limited test {}] log.Log(170): Logging initialized @2384ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T00:28:34,632 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T00:28:34,692 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T00:28:34,711 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T00:28:34,711 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T00:28:34,713 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T00:28:34,733 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T00:28:34,737 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/hadoop.log.dir/,AVAILABLE} 2024-11-13T00:28:34,738 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T00:28:34,917 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@398d2d49{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/java.io.tmpdir/jetty-localhost-39581-hadoop-hdfs-3_4_1-tests_jar-_-any-13458323196463352055/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T00:28:34,923 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:39581} 2024-11-13T00:28:34,924 INFO [Time-limited test {}] server.Server(415): Started @2750ms 2024-11-13T00:28:35,303 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T00:28:35,309 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T00:28:35,310 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T00:28:35,310 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T00:28:35,310 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T00:28:35,311 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/hadoop.log.dir/,AVAILABLE} 2024-11-13T00:28:35,312 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T00:28:35,434 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@54387323{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/java.io.tmpdir/jetty-localhost-34711-hadoop-hdfs-3_4_1-tests_jar-_-any-7851125162621656733/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T00:28:35,435 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:34711} 2024-11-13T00:28:35,435 INFO [Time-limited test {}] server.Server(415): Started @3261ms 2024-11-13T00:28:35,486 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T00:28:35,609 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T00:28:35,616 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T00:28:35,619 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T00:28:35,619 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T00:28:35,619 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T00:28:35,620 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/hadoop.log.dir/,AVAILABLE} 2024-11-13T00:28:35,621 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T00:28:35,748 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7e9308b2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/java.io.tmpdir/jetty-localhost-43953-hadoop-hdfs-3_4_1-tests_jar-_-any-6177296164113130549/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T00:28:35,749 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:43953} 2024-11-13T00:28:35,749 INFO [Time-limited test {}] server.Server(415): Started @3576ms 2024-11-13T00:28:35,752 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T00:28:35,796 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T00:28:35,802 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T00:28:35,805 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T00:28:35,805 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T00:28:35,806 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T00:28:35,806 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/hadoop.log.dir/,AVAILABLE} 2024-11-13T00:28:35,807 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T00:28:35,929 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3217543c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/java.io.tmpdir/jetty-localhost-36951-hadoop-hdfs-3_4_1-tests_jar-_-any-12454000118750764438/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T00:28:35,930 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:36951} 2024-11-13T00:28:35,930 INFO [Time-limited test {}] server.Server(415): Started @3756ms 2024-11-13T00:28:35,934 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T00:28:35,939 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/data/data2/current/BP-1811965340-172.17.0.2-1731457714216/current, will proceed with Du for space computation calculation, 2024-11-13T00:28:35,939 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/data/data3/current/BP-1811965340-172.17.0.2-1731457714216/current, will proceed with Du for space computation calculation, 2024-11-13T00:28:35,939 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/data/data4/current/BP-1811965340-172.17.0.2-1731457714216/current, will proceed with Du for space computation calculation, 2024-11-13T00:28:35,940 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/data/data1/current/BP-1811965340-172.17.0.2-1731457714216/current, will proceed with Du for space computation calculation, 2024-11-13T00:28:35,991 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T00:28:35,991 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T00:28:36,068 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/data/data5/current/BP-1811965340-172.17.0.2-1731457714216/current, will proceed with Du for space computation calculation, 2024-11-13T00:28:36,068 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/data/data6/current/BP-1811965340-172.17.0.2-1731457714216/current, will proceed with Du for space computation calculation, 2024-11-13T00:28:36,079 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7c3c5c3e8208e9e0 with lease ID 0x4daf1bf1fdc531da: Processing first storage report for DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d from datanode DatanodeRegistration(127.0.0.1:40949, datanodeUuid=ac55950c-9dd3-493f-88df-bc11dac90a6f, infoPort=36255, infoSecurePort=0, ipcPort=41605, storageInfo=lv=-57;cid=testClusterID;nsid=820971449;c=1731457714216) 2024-11-13T00:28:36,080 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7c3c5c3e8208e9e0 with lease ID 0x4daf1bf1fdc531da: from storage DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d node DatanodeRegistration(127.0.0.1:40949, datanodeUuid=ac55950c-9dd3-493f-88df-bc11dac90a6f, infoPort=36255, infoSecurePort=0, ipcPort=41605, storageInfo=lv=-57;cid=testClusterID;nsid=820971449;c=1731457714216), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T00:28:36,081 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe7f9eb48d1eee97d with lease ID 0x4daf1bf1fdc531db: Processing first storage report for DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e from datanode DatanodeRegistration(127.0.0.1:41563, datanodeUuid=d618c93e-4a46-4ed3-8257-ff90f02032ae, infoPort=45019, infoSecurePort=0, ipcPort=35453, storageInfo=lv=-57;cid=testClusterID;nsid=820971449;c=1731457714216) 2024-11-13T00:28:36,081 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe7f9eb48d1eee97d with lease ID 0x4daf1bf1fdc531db: from storage DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e node DatanodeRegistration(127.0.0.1:41563, datanodeUuid=d618c93e-4a46-4ed3-8257-ff90f02032ae, infoPort=45019, infoSecurePort=0, ipcPort=35453, storageInfo=lv=-57;cid=testClusterID;nsid=820971449;c=1731457714216), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T00:28:36,082 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7c3c5c3e8208e9e0 with lease ID 0x4daf1bf1fdc531da: Processing first storage report for DS-36227fb7-7415-467c-a5f7-ba9ab9de6016 from datanode DatanodeRegistration(127.0.0.1:40949, datanodeUuid=ac55950c-9dd3-493f-88df-bc11dac90a6f, infoPort=36255, infoSecurePort=0, ipcPort=41605, storageInfo=lv=-57;cid=testClusterID;nsid=820971449;c=1731457714216) 2024-11-13T00:28:36,082 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7c3c5c3e8208e9e0 with lease ID 0x4daf1bf1fdc531da: from storage DS-36227fb7-7415-467c-a5f7-ba9ab9de6016 node DatanodeRegistration(127.0.0.1:40949, datanodeUuid=ac55950c-9dd3-493f-88df-bc11dac90a6f, infoPort=36255, infoSecurePort=0, ipcPort=41605, storageInfo=lv=-57;cid=testClusterID;nsid=820971449;c=1731457714216), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T00:28:36,084 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe7f9eb48d1eee97d with lease ID 0x4daf1bf1fdc531db: Processing first storage report for DS-a045e554-82f7-4492-8ea5-c1648eec3351 from datanode DatanodeRegistration(127.0.0.1:41563, datanodeUuid=d618c93e-4a46-4ed3-8257-ff90f02032ae, infoPort=45019, infoSecurePort=0, ipcPort=35453, storageInfo=lv=-57;cid=testClusterID;nsid=820971449;c=1731457714216) 2024-11-13T00:28:36,085 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe7f9eb48d1eee97d with lease ID 0x4daf1bf1fdc531db: from storage DS-a045e554-82f7-4492-8ea5-c1648eec3351 node DatanodeRegistration(127.0.0.1:41563, datanodeUuid=d618c93e-4a46-4ed3-8257-ff90f02032ae, infoPort=45019, infoSecurePort=0, ipcPort=35453, storageInfo=lv=-57;cid=testClusterID;nsid=820971449;c=1731457714216), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T00:28:36,109 WARN [Thread-119 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T00:28:36,114 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x460a6351d345c96c with lease ID 0x4daf1bf1fdc531dc: Processing first storage report for DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34 from datanode DatanodeRegistration(127.0.0.1:36087, datanodeUuid=e923ddbf-6d36-4714-8dbb-0d1a2e2b0299, infoPort=41441, infoSecurePort=0, ipcPort=35945, storageInfo=lv=-57;cid=testClusterID;nsid=820971449;c=1731457714216) 2024-11-13T00:28:36,115 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x460a6351d345c96c with lease ID 0x4daf1bf1fdc531dc: from storage DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34 node DatanodeRegistration(127.0.0.1:36087, datanodeUuid=e923ddbf-6d36-4714-8dbb-0d1a2e2b0299, infoPort=41441, infoSecurePort=0, ipcPort=35945, storageInfo=lv=-57;cid=testClusterID;nsid=820971449;c=1731457714216), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T00:28:36,115 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x460a6351d345c96c with lease ID 0x4daf1bf1fdc531dc: Processing first storage report for DS-049c5d9b-6879-4261-b245-9385163a6f1a from datanode DatanodeRegistration(127.0.0.1:36087, datanodeUuid=e923ddbf-6d36-4714-8dbb-0d1a2e2b0299, infoPort=41441, infoSecurePort=0, ipcPort=35945, storageInfo=lv=-57;cid=testClusterID;nsid=820971449;c=1731457714216) 2024-11-13T00:28:36,115 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x460a6351d345c96c with lease ID 0x4daf1bf1fdc531dc: from storage DS-049c5d9b-6879-4261-b245-9385163a6f1a node DatanodeRegistration(127.0.0.1:36087, datanodeUuid=e923ddbf-6d36-4714-8dbb-0d1a2e2b0299, infoPort=41441, infoSecurePort=0, ipcPort=35945, storageInfo=lv=-57;cid=testClusterID;nsid=820971449;c=1731457714216), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T00:28:36,328 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d 2024-11-13T00:28:36,412 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/zookeeper_0, clientPort=49723, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T00:28:36,435 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49723 2024-11-13T00:28:36,453 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T00:28:36,456 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T00:28:36,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741825_1001 (size=7) 2024-11-13T00:28:36,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741825_1001 (size=7) 2024-11-13T00:28:36,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741825_1001 (size=7) 2024-11-13T00:28:37,073 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d with version=8 2024-11-13T00:28:37,074 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/hbase-staging 2024-11-13T00:28:37,392 INFO [Time-limited test {}] client.ConnectionUtils(128): master/cb28580903fb:0 server-side Connection retries=45 2024-11-13T00:28:37,403 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T00:28:37,403 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T00:28:37,408 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T00:28:37,408 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T00:28:37,408 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T00:28:37,545 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T00:28:37,605 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T00:28:37,613 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T00:28:37,617 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T00:28:37,645 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 18971 (auto-detected) 2024-11-13T00:28:37,646 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-13T00:28:37,666 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44487 2024-11-13T00:28:37,687 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44487 connecting to ZooKeeper ensemble=127.0.0.1:49723 2024-11-13T00:28:37,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:444870x0, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T00:28:37,720 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44487-0x1010eb9fb3c0000 connected 2024-11-13T00:28:37,752 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T00:28:37,757 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T00:28:37,771 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T00:28:37,776 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d, hbase.cluster.distributed=false 2024-11-13T00:28:37,801 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T00:28:37,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44487 2024-11-13T00:28:37,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44487 2024-11-13T00:28:37,807 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44487 2024-11-13T00:28:37,807 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44487 2024-11-13T00:28:37,807 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44487 2024-11-13T00:28:37,928 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/cb28580903fb:0 server-side Connection retries=45 2024-11-13T00:28:37,930 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T00:28:37,930 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T00:28:37,930 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T00:28:37,931 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T00:28:37,931 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T00:28:37,935 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T00:28:37,938 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T00:28:37,938 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39009 2024-11-13T00:28:37,940 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39009 connecting to ZooKeeper ensemble=127.0.0.1:49723 2024-11-13T00:28:37,942 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T00:28:37,944 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T00:28:37,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:390090x0, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T00:28:37,950 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:390090x0, quorum=127.0.0.1:49723, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T00:28:37,950 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39009-0x1010eb9fb3c0001 connected 2024-11-13T00:28:37,954 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T00:28:37,961 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T00:28:37,963 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39009-0x1010eb9fb3c0001, quorum=127.0.0.1:49723, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T00:28:37,968 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39009-0x1010eb9fb3c0001, quorum=127.0.0.1:49723, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T00:28:37,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39009 2024-11-13T00:28:37,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39009 2024-11-13T00:28:37,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39009 2024-11-13T00:28:37,970 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39009 2024-11-13T00:28:37,970 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39009 2024-11-13T00:28:37,987 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/cb28580903fb:0 server-side Connection retries=45 2024-11-13T00:28:37,987 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T00:28:37,987 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T00:28:37,988 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T00:28:37,988 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T00:28:37,988 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T00:28:37,988 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T00:28:37,988 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T00:28:37,989 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38607 2024-11-13T00:28:37,991 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38607 connecting to ZooKeeper ensemble=127.0.0.1:49723 2024-11-13T00:28:37,992 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T00:28:37,994 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T00:28:38,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:386070x0, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T00:28:38,002 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:386070x0, quorum=127.0.0.1:49723, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T00:28:38,002 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38607-0x1010eb9fb3c0002 connected 2024-11-13T00:28:38,003 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T00:28:38,004 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T00:28:38,005 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38607-0x1010eb9fb3c0002, quorum=127.0.0.1:49723, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T00:28:38,006 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38607-0x1010eb9fb3c0002, quorum=127.0.0.1:49723, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T00:28:38,007 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38607 2024-11-13T00:28:38,007 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38607 2024-11-13T00:28:38,008 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38607 2024-11-13T00:28:38,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38607 2024-11-13T00:28:38,010 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38607 2024-11-13T00:28:38,028 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/cb28580903fb:0 server-side Connection retries=45 2024-11-13T00:28:38,028 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T00:28:38,029 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T00:28:38,029 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T00:28:38,029 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T00:28:38,029 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T00:28:38,029 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T00:28:38,029 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T00:28:38,030 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38771 2024-11-13T00:28:38,032 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38771 connecting to ZooKeeper ensemble=127.0.0.1:49723 2024-11-13T00:28:38,033 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T00:28:38,035 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T00:28:38,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387710x0, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T00:28:38,042 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:387710x0, quorum=127.0.0.1:49723, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T00:28:38,042 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38771-0x1010eb9fb3c0003 connected 2024-11-13T00:28:38,042 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T00:28:38,045 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T00:28:38,045 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38771-0x1010eb9fb3c0003, quorum=127.0.0.1:49723, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T00:28:38,047 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38771-0x1010eb9fb3c0003, quorum=127.0.0.1:49723, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T00:28:38,048 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38771 2024-11-13T00:28:38,048 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38771 2024-11-13T00:28:38,048 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38771 2024-11-13T00:28:38,049 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38771 2024-11-13T00:28:38,049 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38771 2024-11-13T00:28:38,065 DEBUG [M:0;cb28580903fb:44487 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;cb28580903fb:44487 2024-11-13T00:28:38,066 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/cb28580903fb,44487,1731457717196 2024-11-13T00:28:38,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x1010eb9fb3c0002, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T00:28:38,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39009-0x1010eb9fb3c0001, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T00:28:38,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T00:28:38,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38771-0x1010eb9fb3c0003, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T00:28:38,075 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/cb28580903fb,44487,1731457717196 2024-11-13T00:28:38,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38771-0x1010eb9fb3c0003, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T00:28:38,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x1010eb9fb3c0002, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T00:28:38,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:38,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38771-0x1010eb9fb3c0003, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:38,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x1010eb9fb3c0002, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:38,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39009-0x1010eb9fb3c0001, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T00:28:38,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39009-0x1010eb9fb3c0001, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:38,096 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T00:28:38,098 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/cb28580903fb,44487,1731457717196 from backup master directory 2024-11-13T00:28:38,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/cb28580903fb,44487,1731457717196 2024-11-13T00:28:38,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38771-0x1010eb9fb3c0003, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T00:28:38,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T00:28:38,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39009-0x1010eb9fb3c0001, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T00:28:38,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x1010eb9fb3c0002, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T00:28:38,104 WARN [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T00:28:38,105 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=cb28580903fb,44487,1731457717196 2024-11-13T00:28:38,108 INFO [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T00:28:38,109 INFO [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T00:28:38,164 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/hbase.id] with ID: 039f7df9-b54c-4a07-b722-53d63e7e97fb 2024-11-13T00:28:38,165 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/.tmp/hbase.id 2024-11-13T00:28:38,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741826_1002 (size=42) 2024-11-13T00:28:38,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741826_1002 (size=42) 2024-11-13T00:28:38,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741826_1002 (size=42) 2024-11-13T00:28:38,179 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/.tmp/hbase.id]:[hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/hbase.id] 2024-11-13T00:28:38,221 INFO [master/cb28580903fb:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T00:28:38,226 INFO [master/cb28580903fb:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T00:28:38,244 INFO [master/cb28580903fb:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-13T00:28:38,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38771-0x1010eb9fb3c0003, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:38,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39009-0x1010eb9fb3c0001, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:38,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x1010eb9fb3c0002, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:38,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:38,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741827_1003 (size=196) 2024-11-13T00:28:38,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741827_1003 (size=196) 2024-11-13T00:28:38,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741827_1003 (size=196) 2024-11-13T00:28:38,282 INFO [master/cb28580903fb:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T00:28:38,283 INFO [master/cb28580903fb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T00:28:38,290 INFO [master/cb28580903fb:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:38,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741828_1004 (size=1189) 2024-11-13T00:28:38,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741828_1004 (size=1189) 2024-11-13T00:28:38,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741828_1004 (size=1189) 2024-11-13T00:28:38,340 INFO [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/data/master/store 2024-11-13T00:28:38,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741829_1005 (size=34) 2024-11-13T00:28:38,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741829_1005 (size=34) 2024-11-13T00:28:38,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741829_1005 (size=34) 2024-11-13T00:28:38,366 INFO [master/cb28580903fb:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T00:28:38,369 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:38,370 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T00:28:38,370 INFO [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T00:28:38,371 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T00:28:38,372 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T00:28:38,372 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T00:28:38,372 INFO [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T00:28:38,373 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731457718370Disabling compacts and flushes for region at 1731457718370Disabling writes for close at 1731457718372 (+2 ms)Writing region close event to WAL at 1731457718372Closed at 1731457718372 2024-11-13T00:28:38,375 WARN [master/cb28580903fb:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/data/master/store/.initializing 2024-11-13T00:28:38,375 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/WALs/cb28580903fb,44487,1731457717196 2024-11-13T00:28:38,383 INFO [master/cb28580903fb:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T00:28:38,398 INFO [master/cb28580903fb:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cb28580903fb%2C44487%2C1731457717196, suffix=, logDir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/WALs/cb28580903fb,44487,1731457717196, archiveDir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/oldWALs, maxLogs=10 2024-11-13T00:28:38,429 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/WALs/cb28580903fb,44487,1731457717196/cb28580903fb%2C44487%2C1731457717196.1731457718402, exclude list is [], retry=0 2024-11-13T00:28:38,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:28:38,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:38,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:38,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:38,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T00:28:38,496 INFO [master/cb28580903fb:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/WALs/cb28580903fb,44487,1731457717196/cb28580903fb%2C44487%2C1731457717196.1731457718402 2024-11-13T00:28:38,497 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:41441:41441)] 2024-11-13T00:28:38,498 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:28:38,498 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:38,502 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:28:38,503 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:28:38,540 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:28:38,566 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T00:28:38,570 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:38,572 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T00:28:38,573 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:28:38,576 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T00:28:38,576 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:38,577 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:38,577 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:28:38,580 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T00:28:38,580 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:38,581 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:38,581 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:28:38,583 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T00:28:38,583 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:38,584 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:38,585 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:28:38,588 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:28:38,589 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:28:38,594 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:28:38,594 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:28:38,597 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T00:28:38,601 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T00:28:38,605 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:28:38,606 INFO [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61545235, jitterRate=-0.08290453255176544}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T00:28:38,613 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731457718515Initializing all the Stores at 1731457718517 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731457718517Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457718518 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457718518Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457718518Cleaning up temporary data from old regions at 1731457718594 (+76 ms)Region opened successfully at 1731457718612 (+18 ms) 2024-11-13T00:28:38,614 INFO [master/cb28580903fb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T00:28:38,646 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e547d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cb28580903fb/172.17.0.2:0 2024-11-13T00:28:38,677 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T00:28:38,689 INFO [master/cb28580903fb:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T00:28:38,689 INFO [master/cb28580903fb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T00:28:38,692 INFO [master/cb28580903fb:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T00:28:38,693 INFO [master/cb28580903fb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T00:28:38,698 INFO [master/cb28580903fb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T00:28:38,698 INFO [master/cb28580903fb:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T00:28:38,728 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T00:28:38,738 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T00:28:38,740 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T00:28:38,742 INFO [master/cb28580903fb:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T00:28:38,744 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T00:28:38,746 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T00:28:38,748 INFO [master/cb28580903fb:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T00:28:38,752 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T00:28:38,755 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T00:28:38,757 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T00:28:38,758 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T00:28:38,775 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T00:28:38,777 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T00:28:38,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T00:28:38,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39009-0x1010eb9fb3c0001, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T00:28:38,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38771-0x1010eb9fb3c0003, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T00:28:38,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x1010eb9fb3c0002, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T00:28:38,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:38,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39009-0x1010eb9fb3c0001, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:38,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x1010eb9fb3c0002, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:38,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38771-0x1010eb9fb3c0003, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:38,784 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=cb28580903fb,44487,1731457717196, sessionid=0x1010eb9fb3c0000, setting cluster-up flag (Was=false) 2024-11-13T00:28:38,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38771-0x1010eb9fb3c0003, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:38,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:38,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x1010eb9fb3c0002, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:38,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39009-0x1010eb9fb3c0001, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:38,804 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T00:28:38,806 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=cb28580903fb,44487,1731457717196 2024-11-13T00:28:38,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x1010eb9fb3c0002, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:38,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38771-0x1010eb9fb3c0003, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:38,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39009-0x1010eb9fb3c0001, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:38,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:38,818 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T00:28:38,820 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=cb28580903fb,44487,1731457717196 2024-11-13T00:28:38,826 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T00:28:38,853 INFO [RS:1;cb28580903fb:38607 {}] regionserver.HRegionServer(746): ClusterId : 039f7df9-b54c-4a07-b722-53d63e7e97fb 2024-11-13T00:28:38,853 INFO [RS:2;cb28580903fb:38771 {}] regionserver.HRegionServer(746): ClusterId : 039f7df9-b54c-4a07-b722-53d63e7e97fb 2024-11-13T00:28:38,853 INFO [RS:0;cb28580903fb:39009 {}] regionserver.HRegionServer(746): ClusterId : 039f7df9-b54c-4a07-b722-53d63e7e97fb 2024-11-13T00:28:38,856 DEBUG [RS:2;cb28580903fb:38771 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T00:28:38,856 DEBUG [RS:1;cb28580903fb:38607 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T00:28:38,856 DEBUG [RS:0;cb28580903fb:39009 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T00:28:38,861 DEBUG [RS:2;cb28580903fb:38771 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T00:28:38,861 DEBUG [RS:0;cb28580903fb:39009 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T00:28:38,861 DEBUG [RS:1;cb28580903fb:38607 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T00:28:38,861 DEBUG [RS:2;cb28580903fb:38771 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T00:28:38,861 DEBUG [RS:0;cb28580903fb:39009 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T00:28:38,861 DEBUG [RS:1;cb28580903fb:38607 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T00:28:38,866 DEBUG [RS:2;cb28580903fb:38771 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T00:28:38,866 DEBUG [RS:0;cb28580903fb:39009 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T00:28:38,866 DEBUG [RS:1;cb28580903fb:38607 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T00:28:38,867 DEBUG [RS:2;cb28580903fb:38771 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d3845d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cb28580903fb/172.17.0.2:0 2024-11-13T00:28:38,867 DEBUG [RS:0;cb28580903fb:39009 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6159ec0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cb28580903fb/172.17.0.2:0 2024-11-13T00:28:38,867 DEBUG [RS:1;cb28580903fb:38607 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64a8a07b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=cb28580903fb/172.17.0.2:0 2024-11-13T00:28:38,880 DEBUG [RS:2;cb28580903fb:38771 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;cb28580903fb:38771 2024-11-13T00:28:38,883 INFO [RS:2;cb28580903fb:38771 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T00:28:38,883 INFO [RS:2;cb28580903fb:38771 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T00:28:38,883 DEBUG [RS:2;cb28580903fb:38771 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T00:28:38,885 DEBUG [RS:0;cb28580903fb:39009 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;cb28580903fb:39009 2024-11-13T00:28:38,885 INFO [RS:0;cb28580903fb:39009 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T00:28:38,886 INFO [RS:0;cb28580903fb:39009 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T00:28:38,886 DEBUG [RS:0;cb28580903fb:39009 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T00:28:38,886 INFO [RS:2;cb28580903fb:38771 {}] regionserver.HRegionServer(2659): reportForDuty to master=cb28580903fb,44487,1731457717196 with port=38771, startcode=1731457718028 2024-11-13T00:28:38,889 INFO [RS:0;cb28580903fb:39009 {}] regionserver.HRegionServer(2659): reportForDuty to master=cb28580903fb,44487,1731457717196 with port=39009, startcode=1731457717889 2024-11-13T00:28:38,890 DEBUG [RS:1;cb28580903fb:38607 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;cb28580903fb:38607 2024-11-13T00:28:38,890 INFO [RS:1;cb28580903fb:38607 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T00:28:38,890 INFO [RS:1;cb28580903fb:38607 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T00:28:38,890 DEBUG [RS:1;cb28580903fb:38607 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T00:28:38,891 INFO [RS:1;cb28580903fb:38607 {}] regionserver.HRegionServer(2659): reportForDuty to master=cb28580903fb,44487,1731457717196 with port=38607, startcode=1731457717986 2024-11-13T00:28:38,900 DEBUG [RS:0;cb28580903fb:39009 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T00:28:38,900 DEBUG [RS:1;cb28580903fb:38607 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T00:28:38,900 DEBUG [RS:2;cb28580903fb:38771 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T00:28:38,904 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T00:28:38,916 INFO [master/cb28580903fb:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T00:28:38,924 INFO [master/cb28580903fb:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T00:28:38,930 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: cb28580903fb,44487,1731457717196 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T00:28:38,938 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/cb28580903fb:0, corePoolSize=5, maxPoolSize=5 2024-11-13T00:28:38,938 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/cb28580903fb:0, corePoolSize=5, maxPoolSize=5 2024-11-13T00:28:38,938 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/cb28580903fb:0, corePoolSize=5, maxPoolSize=5 2024-11-13T00:28:38,938 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/cb28580903fb:0, corePoolSize=5, maxPoolSize=5 2024-11-13T00:28:38,938 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/cb28580903fb:0, corePoolSize=10, maxPoolSize=10 2024-11-13T00:28:38,939 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:38,939 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/cb28580903fb:0, corePoolSize=2, maxPoolSize=2 2024-11-13T00:28:38,939 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:38,940 INFO [master/cb28580903fb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731457748940 2024-11-13T00:28:38,941 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45249, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T00:28:38,941 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51139, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T00:28:38,941 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41133, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T00:28:38,942 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T00:28:38,944 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T00:28:38,946 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T00:28:38,946 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T00:28:38,947 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T00:28:38,948 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T00:28:38,948 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T00:28:38,948 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T00:28:38,949 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44487 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:28:38,949 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:38,952 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T00:28:38,953 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T00:28:38,953 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:38,953 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T00:28:38,953 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T00:28:38,955 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44487 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:28:38,956 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44487 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T00:28:38,957 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T00:28:38,957 INFO [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T00:28:38,960 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/cb28580903fb:0:becomeActiveMaster-HFileCleaner.large.0-1731457718959,5,FailOnTimeoutGroup] 2024-11-13T00:28:38,960 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/cb28580903fb:0:becomeActiveMaster-HFileCleaner.small.0-1731457718960,5,FailOnTimeoutGroup] 2024-11-13T00:28:38,960 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:38,961 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T00:28:38,962 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:38,962 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:38,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741831_1007 (size=1321) 2024-11-13T00:28:38,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741831_1007 (size=1321) 2024-11-13T00:28:39,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741831_1007 (size=1321) 2024-11-13T00:28:39,001 DEBUG [RS:0;cb28580903fb:39009 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T00:28:39,001 DEBUG [RS:2;cb28580903fb:38771 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T00:28:39,001 DEBUG [RS:1;cb28580903fb:38607 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T00:28:39,002 WARN [RS:1;cb28580903fb:38607 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T00:28:39,002 WARN [RS:2;cb28580903fb:38771 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T00:28:39,002 WARN [RS:0;cb28580903fb:39009 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T00:28:39,002 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T00:28:39,002 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d 2024-11-13T00:28:39,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741832_1008 (size=32) 2024-11-13T00:28:39,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741832_1008 (size=32) 2024-11-13T00:28:39,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741832_1008 (size=32) 2024-11-13T00:28:39,016 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:39,019 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T00:28:39,021 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T00:28:39,021 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:39,022 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T00:28:39,022 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T00:28:39,024 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T00:28:39,024 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:39,025 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T00:28:39,025 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T00:28:39,028 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T00:28:39,028 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:39,029 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T00:28:39,029 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T00:28:39,031 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T00:28:39,031 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:39,032 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T00:28:39,032 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T00:28:39,034 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740 2024-11-13T00:28:39,034 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740 2024-11-13T00:28:39,037 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T00:28:39,037 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T00:28:39,038 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T00:28:39,041 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T00:28:39,045 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:28:39,046 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69906486, jitterRate=0.04168781638145447}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T00:28:39,048 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731457719017Initializing all the Stores at 1731457719018 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731457719018Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731457719018Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457719018Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731457719019 (+1 ms)Cleaning up temporary data from old regions at 1731457719037 (+18 ms)Region opened successfully at 1731457719048 (+11 ms) 2024-11-13T00:28:39,048 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T00:28:39,048 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T00:28:39,049 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T00:28:39,049 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T00:28:39,049 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T00:28:39,050 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T00:28:39,050 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731457719048Disabling compacts and flushes for region at 1731457719048Disabling writes for close at 1731457719049 (+1 ms)Writing region close event to WAL at 1731457719050 (+1 ms)Closed at 1731457719050 2024-11-13T00:28:39,054 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T00:28:39,054 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T00:28:39,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T00:28:39,069 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T00:28:39,073 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T00:28:39,103 INFO [RS:2;cb28580903fb:38771 {}] regionserver.HRegionServer(2659): reportForDuty to master=cb28580903fb,44487,1731457717196 with port=38771, startcode=1731457718028 2024-11-13T00:28:39,103 INFO [RS:1;cb28580903fb:38607 {}] regionserver.HRegionServer(2659): reportForDuty to master=cb28580903fb,44487,1731457717196 with port=38607, startcode=1731457717986 2024-11-13T00:28:39,103 INFO [RS:0;cb28580903fb:39009 {}] regionserver.HRegionServer(2659): reportForDuty to master=cb28580903fb,44487,1731457717196 with port=39009, startcode=1731457717889 2024-11-13T00:28:39,105 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44487 {}] master.ServerManager(363): Checking decommissioned status of RegionServer cb28580903fb,38771,1731457718028 2024-11-13T00:28:39,108 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44487 {}] master.ServerManager(517): Registering regionserver=cb28580903fb,38771,1731457718028 2024-11-13T00:28:39,115 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44487 {}] master.ServerManager(363): Checking decommissioned status of RegionServer cb28580903fb,38607,1731457717986 2024-11-13T00:28:39,116 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44487 {}] master.ServerManager(517): Registering regionserver=cb28580903fb,38607,1731457717986 2024-11-13T00:28:39,116 DEBUG [RS:2;cb28580903fb:38771 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d 2024-11-13T00:28:39,116 DEBUG [RS:2;cb28580903fb:38771 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42745 2024-11-13T00:28:39,116 DEBUG [RS:2;cb28580903fb:38771 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T00:28:39,118 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44487 {}] master.ServerManager(363): Checking decommissioned status of RegionServer cb28580903fb,39009,1731457717889 2024-11-13T00:28:39,118 DEBUG [RS:1;cb28580903fb:38607 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d 2024-11-13T00:28:39,118 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44487 {}] master.ServerManager(517): Registering regionserver=cb28580903fb,39009,1731457717889 2024-11-13T00:28:39,118 DEBUG [RS:1;cb28580903fb:38607 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42745 2024-11-13T00:28:39,118 DEBUG [RS:1;cb28580903fb:38607 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T00:28:39,120 DEBUG [RS:0;cb28580903fb:39009 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d 2024-11-13T00:28:39,120 DEBUG [RS:0;cb28580903fb:39009 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42745 2024-11-13T00:28:39,120 DEBUG [RS:0;cb28580903fb:39009 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T00:28:39,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T00:28:39,126 DEBUG [RS:1;cb28580903fb:38607 {}] zookeeper.ZKUtil(111): regionserver:38607-0x1010eb9fb3c0002, quorum=127.0.0.1:49723, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/cb28580903fb,38607,1731457717986 2024-11-13T00:28:39,126 WARN [RS:1;cb28580903fb:38607 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T00:28:39,126 INFO [RS:1;cb28580903fb:38607 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:39,126 DEBUG [RS:1;cb28580903fb:38607 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38607,1731457717986 2024-11-13T00:28:39,127 DEBUG [RS:2;cb28580903fb:38771 {}] zookeeper.ZKUtil(111): regionserver:38771-0x1010eb9fb3c0003, quorum=127.0.0.1:49723, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/cb28580903fb,38771,1731457718028 2024-11-13T00:28:39,127 WARN [RS:2;cb28580903fb:38771 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T00:28:39,127 INFO [RS:2;cb28580903fb:38771 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:39,127 DEBUG [RS:2;cb28580903fb:38771 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38771,1731457718028 2024-11-13T00:28:39,127 DEBUG [RS:0;cb28580903fb:39009 {}] zookeeper.ZKUtil(111): regionserver:39009-0x1010eb9fb3c0001, quorum=127.0.0.1:49723, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/cb28580903fb,39009,1731457717889 2024-11-13T00:28:39,128 WARN [RS:0;cb28580903fb:39009 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T00:28:39,128 INFO [RS:0;cb28580903fb:39009 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:39,128 DEBUG [RS:0;cb28580903fb:39009 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,39009,1731457717889 2024-11-13T00:28:39,129 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [cb28580903fb,39009,1731457717889] 2024-11-13T00:28:39,129 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [cb28580903fb,38771,1731457718028] 2024-11-13T00:28:39,129 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [cb28580903fb,38607,1731457717986] 2024-11-13T00:28:39,155 INFO [RS:2;cb28580903fb:38771 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T00:28:39,155 INFO [RS:1;cb28580903fb:38607 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T00:28:39,155 INFO [RS:0;cb28580903fb:39009 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T00:28:39,170 INFO [RS:0;cb28580903fb:39009 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T00:28:39,170 INFO [RS:2;cb28580903fb:38771 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T00:28:39,170 INFO [RS:1;cb28580903fb:38607 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T00:28:39,176 INFO [RS:2;cb28580903fb:38771 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T00:28:39,176 INFO [RS:1;cb28580903fb:38607 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T00:28:39,176 INFO [RS:0;cb28580903fb:39009 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T00:28:39,176 INFO [RS:1;cb28580903fb:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,176 INFO [RS:2;cb28580903fb:38771 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,176 INFO [RS:0;cb28580903fb:39009 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,177 INFO [RS:1;cb28580903fb:38607 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T00:28:39,178 INFO [RS:2;cb28580903fb:38771 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T00:28:39,180 INFO [RS:0;cb28580903fb:39009 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T00:28:39,183 INFO [RS:2;cb28580903fb:38771 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T00:28:39,183 INFO [RS:1;cb28580903fb:38607 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T00:28:39,183 INFO [RS:0;cb28580903fb:39009 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T00:28:39,184 INFO [RS:0;cb28580903fb:39009 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,184 INFO [RS:1;cb28580903fb:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,184 INFO [RS:2;cb28580903fb:38771 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,185 DEBUG [RS:1;cb28580903fb:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,185 DEBUG [RS:0;cb28580903fb:39009 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,185 DEBUG [RS:2;cb28580903fb:38771 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,185 DEBUG [RS:0;cb28580903fb:39009 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,185 DEBUG [RS:1;cb28580903fb:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,185 DEBUG [RS:2;cb28580903fb:38771 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,185 DEBUG [RS:0;cb28580903fb:39009 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,185 DEBUG [RS:2;cb28580903fb:38771 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,185 DEBUG [RS:0;cb28580903fb:39009 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,185 DEBUG [RS:1;cb28580903fb:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,185 DEBUG [RS:0;cb28580903fb:39009 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,185 DEBUG [RS:1;cb28580903fb:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,185 DEBUG [RS:2;cb28580903fb:38771 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,185 DEBUG [RS:0;cb28580903fb:39009 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/cb28580903fb:0, corePoolSize=2, maxPoolSize=2 2024-11-13T00:28:39,185 DEBUG [RS:2;cb28580903fb:38771 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,185 DEBUG [RS:1;cb28580903fb:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,185 DEBUG [RS:0;cb28580903fb:39009 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,185 DEBUG [RS:0;cb28580903fb:39009 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,185 DEBUG [RS:2;cb28580903fb:38771 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/cb28580903fb:0, corePoolSize=2, maxPoolSize=2 2024-11-13T00:28:39,185 DEBUG [RS:1;cb28580903fb:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/cb28580903fb:0, corePoolSize=2, maxPoolSize=2 2024-11-13T00:28:39,186 DEBUG [RS:0;cb28580903fb:39009 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,186 DEBUG [RS:1;cb28580903fb:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,186 DEBUG [RS:2;cb28580903fb:38771 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,186 DEBUG [RS:0;cb28580903fb:39009 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,186 DEBUG [RS:2;cb28580903fb:38771 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,186 DEBUG [RS:1;cb28580903fb:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,186 DEBUG [RS:0;cb28580903fb:39009 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,186 DEBUG [RS:0;cb28580903fb:39009 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,186 DEBUG [RS:1;cb28580903fb:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,186 DEBUG [RS:2;cb28580903fb:38771 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,186 DEBUG [RS:0;cb28580903fb:39009 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/cb28580903fb:0, corePoolSize=3, maxPoolSize=3 2024-11-13T00:28:39,186 DEBUG [RS:1;cb28580903fb:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,186 DEBUG [RS:2;cb28580903fb:38771 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,186 DEBUG [RS:0;cb28580903fb:39009 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0, corePoolSize=3, maxPoolSize=3 2024-11-13T00:28:39,186 DEBUG [RS:1;cb28580903fb:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,186 DEBUG [RS:2;cb28580903fb:38771 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,186 DEBUG [RS:1;cb28580903fb:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,186 DEBUG [RS:2;cb28580903fb:38771 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/cb28580903fb:0, corePoolSize=1, maxPoolSize=1 2024-11-13T00:28:39,187 DEBUG [RS:1;cb28580903fb:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/cb28580903fb:0, corePoolSize=3, maxPoolSize=3 2024-11-13T00:28:39,187 DEBUG [RS:2;cb28580903fb:38771 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/cb28580903fb:0, corePoolSize=3, maxPoolSize=3 2024-11-13T00:28:39,187 DEBUG [RS:1;cb28580903fb:38607 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0, corePoolSize=3, maxPoolSize=3 2024-11-13T00:28:39,187 DEBUG [RS:2;cb28580903fb:38771 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/cb28580903fb:0, corePoolSize=3, maxPoolSize=3 2024-11-13T00:28:39,192 INFO [RS:1;cb28580903fb:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,192 INFO [RS:2;cb28580903fb:38771 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,192 INFO [RS:1;cb28580903fb:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,192 INFO [RS:2;cb28580903fb:38771 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,192 INFO [RS:1;cb28580903fb:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,192 INFO [RS:0;cb28580903fb:39009 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,192 INFO [RS:2;cb28580903fb:38771 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,192 INFO [RS:1;cb28580903fb:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,192 INFO [RS:0;cb28580903fb:39009 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,192 INFO [RS:1;cb28580903fb:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,192 INFO [RS:2;cb28580903fb:38771 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,192 INFO [RS:0;cb28580903fb:39009 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,192 INFO [RS:2;cb28580903fb:38771 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,192 INFO [RS:1;cb28580903fb:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=cb28580903fb,38607,1731457717986-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T00:28:39,193 INFO [RS:2;cb28580903fb:38771 {}] hbase.ChoreService(168): Chore ScheduledChore name=cb28580903fb,38771,1731457718028-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T00:28:39,193 INFO [RS:0;cb28580903fb:39009 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,193 INFO [RS:0;cb28580903fb:39009 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,193 INFO [RS:0;cb28580903fb:39009 {}] hbase.ChoreService(168): Chore ScheduledChore name=cb28580903fb,39009,1731457717889-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T00:28:39,213 INFO [RS:0;cb28580903fb:39009 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T00:28:39,213 INFO [RS:2;cb28580903fb:38771 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T00:28:39,214 INFO [RS:1;cb28580903fb:38607 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T00:28:39,215 INFO [RS:2;cb28580903fb:38771 {}] hbase.ChoreService(168): Chore ScheduledChore name=cb28580903fb,38771,1731457718028-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,215 INFO [RS:0;cb28580903fb:39009 {}] hbase.ChoreService(168): Chore ScheduledChore name=cb28580903fb,39009,1731457717889-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,215 INFO [RS:1;cb28580903fb:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=cb28580903fb,38607,1731457717986-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,215 INFO [RS:0;cb28580903fb:39009 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,215 INFO [RS:2;cb28580903fb:38771 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,215 INFO [RS:1;cb28580903fb:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,215 INFO [RS:0;cb28580903fb:39009 {}] regionserver.Replication(171): cb28580903fb,39009,1731457717889 started 2024-11-13T00:28:39,215 INFO [RS:2;cb28580903fb:38771 {}] regionserver.Replication(171): cb28580903fb,38771,1731457718028 started 2024-11-13T00:28:39,215 INFO [RS:1;cb28580903fb:38607 {}] regionserver.Replication(171): cb28580903fb,38607,1731457717986 started 2024-11-13T00:28:39,224 WARN [cb28580903fb:44487 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T00:28:39,236 INFO [RS:0;cb28580903fb:39009 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,236 INFO [RS:2;cb28580903fb:38771 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,236 INFO [RS:2;cb28580903fb:38771 {}] regionserver.HRegionServer(1482): Serving as cb28580903fb,38771,1731457718028, RpcServer on cb28580903fb/172.17.0.2:38771, sessionid=0x1010eb9fb3c0003 2024-11-13T00:28:39,236 INFO [RS:0;cb28580903fb:39009 {}] regionserver.HRegionServer(1482): Serving as cb28580903fb,39009,1731457717889, RpcServer on cb28580903fb/172.17.0.2:39009, sessionid=0x1010eb9fb3c0001 2024-11-13T00:28:39,237 DEBUG [RS:2;cb28580903fb:38771 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T00:28:39,237 DEBUG [RS:0;cb28580903fb:39009 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T00:28:39,237 DEBUG [RS:2;cb28580903fb:38771 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager cb28580903fb,38771,1731457718028 2024-11-13T00:28:39,237 DEBUG [RS:0;cb28580903fb:39009 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager cb28580903fb,39009,1731457717889 2024-11-13T00:28:39,237 DEBUG [RS:0;cb28580903fb:39009 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cb28580903fb,39009,1731457717889' 2024-11-13T00:28:39,237 DEBUG [RS:2;cb28580903fb:38771 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cb28580903fb,38771,1731457718028' 2024-11-13T00:28:39,237 DEBUG [RS:0;cb28580903fb:39009 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T00:28:39,237 DEBUG [RS:2;cb28580903fb:38771 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T00:28:39,238 DEBUG [RS:0;cb28580903fb:39009 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T00:28:39,238 DEBUG [RS:2;cb28580903fb:38771 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T00:28:39,239 DEBUG [RS:0;cb28580903fb:39009 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T00:28:39,239 DEBUG [RS:2;cb28580903fb:38771 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T00:28:39,239 DEBUG [RS:2;cb28580903fb:38771 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T00:28:39,239 DEBUG [RS:0;cb28580903fb:39009 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T00:28:39,239 DEBUG [RS:2;cb28580903fb:38771 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager cb28580903fb,38771,1731457718028 2024-11-13T00:28:39,239 DEBUG [RS:2;cb28580903fb:38771 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cb28580903fb,38771,1731457718028' 2024-11-13T00:28:39,239 DEBUG [RS:0;cb28580903fb:39009 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager cb28580903fb,39009,1731457717889 2024-11-13T00:28:39,239 DEBUG [RS:2;cb28580903fb:38771 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T00:28:39,239 DEBUG [RS:0;cb28580903fb:39009 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cb28580903fb,39009,1731457717889' 2024-11-13T00:28:39,239 DEBUG [RS:0;cb28580903fb:39009 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T00:28:39,240 DEBUG [RS:0;cb28580903fb:39009 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T00:28:39,240 DEBUG [RS:2;cb28580903fb:38771 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T00:28:39,241 DEBUG [RS:2;cb28580903fb:38771 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T00:28:39,241 DEBUG [RS:0;cb28580903fb:39009 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T00:28:39,241 INFO [RS:2;cb28580903fb:38771 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T00:28:39,241 INFO [RS:0;cb28580903fb:39009 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T00:28:39,241 INFO [RS:2;cb28580903fb:38771 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T00:28:39,241 INFO [RS:0;cb28580903fb:39009 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T00:28:39,241 INFO [RS:1;cb28580903fb:38607 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,241 INFO [RS:1;cb28580903fb:38607 {}] regionserver.HRegionServer(1482): Serving as cb28580903fb,38607,1731457717986, RpcServer on cb28580903fb/172.17.0.2:38607, sessionid=0x1010eb9fb3c0002 2024-11-13T00:28:39,241 DEBUG [RS:1;cb28580903fb:38607 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T00:28:39,241 DEBUG [RS:1;cb28580903fb:38607 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager cb28580903fb,38607,1731457717986 2024-11-13T00:28:39,242 DEBUG [RS:1;cb28580903fb:38607 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cb28580903fb,38607,1731457717986' 2024-11-13T00:28:39,242 DEBUG [RS:1;cb28580903fb:38607 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T00:28:39,242 DEBUG [RS:1;cb28580903fb:38607 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T00:28:39,243 DEBUG [RS:1;cb28580903fb:38607 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T00:28:39,243 DEBUG [RS:1;cb28580903fb:38607 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T00:28:39,243 DEBUG [RS:1;cb28580903fb:38607 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager cb28580903fb,38607,1731457717986 2024-11-13T00:28:39,243 DEBUG [RS:1;cb28580903fb:38607 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'cb28580903fb,38607,1731457717986' 2024-11-13T00:28:39,243 DEBUG [RS:1;cb28580903fb:38607 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T00:28:39,244 DEBUG [RS:1;cb28580903fb:38607 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T00:28:39,244 DEBUG [RS:1;cb28580903fb:38607 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T00:28:39,245 INFO [RS:1;cb28580903fb:38607 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T00:28:39,245 INFO [RS:1;cb28580903fb:38607 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T00:28:39,347 INFO [RS:2;cb28580903fb:38771 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T00:28:39,347 INFO [RS:0;cb28580903fb:39009 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T00:28:39,347 INFO [RS:1;cb28580903fb:38607 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T00:28:39,350 INFO [RS:2;cb28580903fb:38771 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cb28580903fb%2C38771%2C1731457718028, suffix=, logDir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38771,1731457718028, archiveDir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/oldWALs, maxLogs=32 2024-11-13T00:28:39,350 INFO [RS:1;cb28580903fb:38607 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cb28580903fb%2C38607%2C1731457717986, suffix=, logDir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38607,1731457717986, archiveDir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/oldWALs, maxLogs=32 2024-11-13T00:28:39,350 INFO [RS:0;cb28580903fb:39009 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cb28580903fb%2C39009%2C1731457717889, suffix=, logDir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,39009,1731457717889, archiveDir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/oldWALs, maxLogs=32 2024-11-13T00:28:39,370 DEBUG [RS:2;cb28580903fb:38771 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38771,1731457718028/cb28580903fb%2C38771%2C1731457718028.1731457719355, exclude list is [], retry=0 2024-11-13T00:28:39,370 DEBUG [RS:1;cb28580903fb:38607 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38607,1731457717986/cb28580903fb%2C38607%2C1731457717986.1731457719355, exclude list is [], retry=0 2024-11-13T00:28:39,370 DEBUG [RS:0;cb28580903fb:39009 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,39009,1731457717889/cb28580903fb%2C39009%2C1731457717889.1731457719355, exclude list is [], retry=0 2024-11-13T00:28:39,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:39,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:39,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:39,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:39,376 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:39,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:39,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:39,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:39,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:39,383 INFO [RS:2;cb28580903fb:38771 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38771,1731457718028/cb28580903fb%2C38771%2C1731457718028.1731457719355 2024-11-13T00:28:39,384 INFO [RS:1;cb28580903fb:38607 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38607,1731457717986/cb28580903fb%2C38607%2C1731457717986.1731457719355 2024-11-13T00:28:39,384 DEBUG [RS:2;cb28580903fb:38771 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:36255:36255)] 2024-11-13T00:28:39,384 INFO [RS:0;cb28580903fb:39009 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,39009,1731457717889/cb28580903fb%2C39009%2C1731457717889.1731457719355 2024-11-13T00:28:39,384 DEBUG [RS:1;cb28580903fb:38607 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:36255:36255)] 2024-11-13T00:28:39,384 DEBUG [RS:0;cb28580903fb:39009 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:41441:41441)] 2024-11-13T00:28:39,476 DEBUG [cb28580903fb:44487 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-13T00:28:39,485 DEBUG [cb28580903fb:44487 {}] balancer.BalancerClusterState(204): Hosts are {cb28580903fb=0} racks are {/default-rack=0} 2024-11-13T00:28:39,491 DEBUG [cb28580903fb:44487 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T00:28:39,491 DEBUG [cb28580903fb:44487 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T00:28:39,491 DEBUG [cb28580903fb:44487 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T00:28:39,491 DEBUG [cb28580903fb:44487 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T00:28:39,491 DEBUG [cb28580903fb:44487 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T00:28:39,491 DEBUG [cb28580903fb:44487 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T00:28:39,492 INFO [cb28580903fb:44487 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T00:28:39,492 INFO [cb28580903fb:44487 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T00:28:39,492 INFO [cb28580903fb:44487 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T00:28:39,492 DEBUG [cb28580903fb:44487 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T00:28:39,499 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=cb28580903fb,39009,1731457717889 2024-11-13T00:28:39,506 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as cb28580903fb,39009,1731457717889, state=OPENING 2024-11-13T00:28:39,511 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T00:28:39,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38771-0x1010eb9fb3c0003, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:39,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39009-0x1010eb9fb3c0001, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:39,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:39,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x1010eb9fb3c0002, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:28:39,514 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T00:28:39,514 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T00:28:39,514 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T00:28:39,514 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T00:28:39,516 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T00:28:39,517 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=cb28580903fb,39009,1731457717889}] 2024-11-13T00:28:39,694 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T00:28:39,697 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39739, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T00:28:39,709 INFO [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T00:28:39,709 INFO [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:39,710 INFO [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T00:28:39,713 INFO [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=cb28580903fb%2C39009%2C1731457717889.meta, suffix=.meta, logDir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,39009,1731457717889, archiveDir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/oldWALs, maxLogs=32 2024-11-13T00:28:39,729 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,39009,1731457717889/cb28580903fb%2C39009%2C1731457717889.meta.1731457719714.meta, exclude list is [], retry=0 2024-11-13T00:28:39,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:39,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:39,733 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:39,736 INFO [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,39009,1731457717889/cb28580903fb%2C39009%2C1731457717889.meta.1731457719714.meta 2024-11-13T00:28:39,740 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:36255:36255)] 2024-11-13T00:28:39,741 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:28:39,742 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T00:28:39,745 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T00:28:39,750 INFO [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T00:28:39,754 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T00:28:39,755 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:39,755 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T00:28:39,755 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T00:28:39,758 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T00:28:39,760 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T00:28:39,760 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:39,761 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T00:28:39,761 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T00:28:39,763 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T00:28:39,763 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:39,764 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T00:28:39,765 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T00:28:39,766 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T00:28:39,766 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:39,767 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T00:28:39,767 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T00:28:39,768 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T00:28:39,768 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:39,769 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T00:28:39,769 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T00:28:39,771 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740 2024-11-13T00:28:39,773 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740 2024-11-13T00:28:39,775 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T00:28:39,775 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T00:28:39,776 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T00:28:39,779 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T00:28:39,780 INFO [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73365322, jitterRate=0.09322848916053772}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T00:28:39,780 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T00:28:39,782 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731457719755Writing region info on filesystem at 1731457719756 (+1 ms)Initializing all the Stores at 1731457719758 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731457719758Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731457719758Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457719758Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731457719758Cleaning up temporary data from old regions at 1731457719775 (+17 ms)Running coprocessor post-open hooks at 1731457719780 (+5 ms)Region opened successfully at 1731457719782 (+2 ms) 2024-11-13T00:28:39,789 INFO [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731457719684 2024-11-13T00:28:39,800 DEBUG [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T00:28:39,800 INFO [RS_OPEN_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T00:28:39,802 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=cb28580903fb,39009,1731457717889 2024-11-13T00:28:39,804 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as cb28580903fb,39009,1731457717889, state=OPEN 2024-11-13T00:28:39,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38771-0x1010eb9fb3c0003, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T00:28:39,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39009-0x1010eb9fb3c0001, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T00:28:39,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T00:28:39,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x1010eb9fb3c0002, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T00:28:39,808 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T00:28:39,808 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T00:28:39,808 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T00:28:39,808 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T00:28:39,808 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=cb28580903fb,39009,1731457717889 2024-11-13T00:28:39,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T00:28:39,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=cb28580903fb,39009,1731457717889 in 291 msec 2024-11-13T00:28:39,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T00:28:39,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 757 msec 2024-11-13T00:28:39,825 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T00:28:39,825 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T00:28:39,846 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:28:39,848 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,39009,1731457717889, seqNum=-1] 2024-11-13T00:28:39,876 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:28:39,879 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49967, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:28:39,902 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0470 sec 2024-11-13T00:28:39,902 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731457719902, completionTime=-1 2024-11-13T00:28:39,906 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-13T00:28:39,906 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T00:28:39,966 INFO [master/cb28580903fb:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-13T00:28:39,966 INFO [master/cb28580903fb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731457779966 2024-11-13T00:28:39,966 INFO [master/cb28580903fb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731457839966 2024-11-13T00:28:39,966 INFO [master/cb28580903fb:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 60 msec 2024-11-13T00:28:39,968 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-13T00:28:39,977 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cb28580903fb,44487,1731457717196-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,978 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cb28580903fb,44487,1731457717196-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,978 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cb28580903fb,44487,1731457717196-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,980 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-cb28580903fb:44487, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,981 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,981 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:39,992 DEBUG [master/cb28580903fb:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T00:28:40,013 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.907sec 2024-11-13T00:28:40,014 INFO [master/cb28580903fb:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T00:28:40,015 INFO [master/cb28580903fb:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T00:28:40,016 INFO [master/cb28580903fb:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T00:28:40,017 INFO [master/cb28580903fb:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T00:28:40,017 INFO [master/cb28580903fb:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T00:28:40,018 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cb28580903fb,44487,1731457717196-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T00:28:40,018 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cb28580903fb,44487,1731457717196-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T00:28:40,023 DEBUG [master/cb28580903fb:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T00:28:40,024 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T00:28:40,025 INFO [master/cb28580903fb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=cb28580903fb,44487,1731457717196-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:40,067 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c7df72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:28:40,068 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request cb28580903fb,44487,-1 for getting cluster id 2024-11-13T00:28:40,071 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T00:28:40,079 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '039f7df9-b54c-4a07-b722-53d63e7e97fb' 2024-11-13T00:28:40,081 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T00:28:40,081 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "039f7df9-b54c-4a07-b722-53d63e7e97fb" 2024-11-13T00:28:40,083 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bbc2c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:28:40,083 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [cb28580903fb,44487,-1] 2024-11-13T00:28:40,086 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T00:28:40,087 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:28:40,089 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T00:28:40,092 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c11b5a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T00:28:40,093 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T00:28:40,101 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=cb28580903fb,39009,1731457717889, seqNum=-1] 2024-11-13T00:28:40,101 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:28:40,104 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:28:40,125 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=cb28580903fb,44487,1731457717196 2024-11-13T00:28:40,126 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:42745/hbase 2024-11-13T00:28:40,140 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=365, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=116, ProcessCount=11, AvailableMemoryMB=1950 2024-11-13T00:28:40,159 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:40,161 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:40,162 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T00:28:40,166 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-84825553, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/hregion-84825553, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:28:40,181 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-84825553/hregion-84825553.1731457720168, exclude list is [], retry=0 2024-11-13T00:28:40,186 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:40,186 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:40,186 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:40,190 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-84825553/hregion-84825553.1731457720168 2024-11-13T00:28:40,191 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:45019:45019)] 2024-11-13T00:28:40,191 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 687f50a2a27f6c55561cea830ccd0689, NAME => 'testReplayEditsWrittenIntoWAL,,1731457720159.687f50a2a27f6c55561cea830ccd0689.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42745/hbase 2024-11-13T00:28:40,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741838_1014 (size=64) 2024-11-13T00:28:40,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741838_1014 (size=64) 2024-11-13T00:28:40,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741838_1014 (size=64) 2024-11-13T00:28:40,209 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731457720159.687f50a2a27f6c55561cea830ccd0689.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:40,212 INFO [StoreOpener-687f50a2a27f6c55561cea830ccd0689-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 687f50a2a27f6c55561cea830ccd0689 2024-11-13T00:28:40,215 INFO [StoreOpener-687f50a2a27f6c55561cea830ccd0689-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 687f50a2a27f6c55561cea830ccd0689 columnFamilyName a 2024-11-13T00:28:40,215 DEBUG [StoreOpener-687f50a2a27f6c55561cea830ccd0689-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:40,216 INFO [StoreOpener-687f50a2a27f6c55561cea830ccd0689-1 {}] regionserver.HStore(327): Store=687f50a2a27f6c55561cea830ccd0689/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:40,216 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 687f50a2a27f6c55561cea830ccd0689 2024-11-13T00:28:40,218 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689 2024-11-13T00:28:40,219 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689 2024-11-13T00:28:40,219 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 687f50a2a27f6c55561cea830ccd0689 2024-11-13T00:28:40,219 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 687f50a2a27f6c55561cea830ccd0689 2024-11-13T00:28:40,223 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 687f50a2a27f6c55561cea830ccd0689 2024-11-13T00:28:40,227 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:28:40,228 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 687f50a2a27f6c55561cea830ccd0689; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75418002, jitterRate=0.12381580471992493}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T00:28:40,229 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 687f50a2a27f6c55561cea830ccd0689: Writing region info on filesystem at 1731457720209Initializing all the Stores at 1731457720211 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457720211Cleaning up temporary data from old regions at 1731457720219 (+8 ms)Region opened successfully at 1731457720229 (+10 ms) 2024-11-13T00:28:40,229 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 687f50a2a27f6c55561cea830ccd0689, disabling compactions & flushes 2024-11-13T00:28:40,229 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731457720159.687f50a2a27f6c55561cea830ccd0689. 2024-11-13T00:28:40,229 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731457720159.687f50a2a27f6c55561cea830ccd0689. 2024-11-13T00:28:40,229 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731457720159.687f50a2a27f6c55561cea830ccd0689. after waiting 0 ms 2024-11-13T00:28:40,229 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731457720159.687f50a2a27f6c55561cea830ccd0689. 2024-11-13T00:28:40,231 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731457720159.687f50a2a27f6c55561cea830ccd0689. 2024-11-13T00:28:40,231 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 687f50a2a27f6c55561cea830ccd0689: Waiting for close lock at 1731457720229Disabling compacts and flushes for region at 1731457720229Disabling writes for close at 1731457720229Writing region close event to WAL at 1731457720230 (+1 ms)Closed at 1731457720230 2024-11-13T00:28:40,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741837_1013 (size=93) 2024-11-13T00:28:40,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741837_1013 (size=93) 2024-11-13T00:28:40,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741837_1013 (size=93) 2024-11-13T00:28:40,246 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T00:28:40,246 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-84825553:(num 1731457720168) 2024-11-13T00:28:40,248 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T00:28:40,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741839_1015 (size=369) 2024-11-13T00:28:40,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741839_1015 (size=369) 2024-11-13T00:28:40,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741839_1015 (size=369) 2024-11-13T00:28:40,268 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T00:28:40,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741840_1016 (size=231) 2024-11-13T00:28:40,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741840_1016 (size=231) 2024-11-13T00:28:40,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741840_1016 (size=231) 2024-11-13T00:28:40,304 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal-1, size=369 (369bytes) 2024-11-13T00:28:40,305 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-13T00:28:40,305 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-13T00:28:40,305 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal-1 2024-11-13T00:28:40,311 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal-1 after 4ms 2024-11-13T00:28:40,317 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal-1 took 14ms 2024-11-13T00:28:40,321 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal-1 so closing down 2024-11-13T00:28:40,324 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-13T00:28:40,326 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/recovered.edits/0000000000000000001-wal-1.temp 2024-11-13T00:28:40,326 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T00:28:40,327 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T00:28:40,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741841_1017 (size=369) 2024-11-13T00:28:40,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741841_1017 (size=369) 2024-11-13T00:28:40,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741841_1017 (size=369) 2024-11-13T00:28:40,341 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T00:28:40,345 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/recovered.edits/0000000000000000002 2024-11-13T00:28:40,348 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 27 ms; skipped=0; WAL=hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-13T00:28:40,348 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal-1, journal: Splitting hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal-1, size=369 (369bytes) at 1731457720304Finishing writing output for hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal-1 so closing down at 1731457720321 (+17 ms)Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/recovered.edits/0000000000000000001-wal-1.temp at 1731457720326 (+5 ms)3 split writer threads finished at 1731457720327 (+1 ms)Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731457720342 (+15 ms)Rename recovered edits hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/recovered.edits/0000000000000000002 at 1731457720345 (+3 ms)Processed 2 edits across 1 Regions in 27 ms; skipped=0; WAL=hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1731457720348 (+3 ms) 2024-11-13T00:28:40,363 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal-2, size=231 (231bytes) 2024-11-13T00:28:40,363 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal-2 2024-11-13T00:28:40,364 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal-2 after 1ms 2024-11-13T00:28:40,368 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal-2 took 5ms 2024-11-13T00:28:40,371 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal-2 so closing down 2024-11-13T00:28:40,371 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T00:28:40,373 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-13T00:28:40,375 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/recovered.edits/0000000000000000002-wal-2.temp 2024-11-13T00:28:40,375 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T00:28:40,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741842_1018 (size=231) 2024-11-13T00:28:40,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741842_1018 (size=231) 2024-11-13T00:28:40,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741842_1018 (size=231) 2024-11-13T00:28:40,384 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T00:28:40,391 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-11-13T00:28:40,392 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-13T00:28:40,392 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal-2, journal: Splitting hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal-2, size=231 (231bytes) at 1731457720363Finishing writing output for hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal-2 so closing down at 1731457720371 (+8 ms)Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/recovered.edits/0000000000000000002-wal-2.temp at 1731457720375 (+4 ms)3 split writer threads finished at 1731457720375Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731457720384 (+9 ms)Processed 1 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1731457720392 (+8 ms) 2024-11-13T00:28:40,393 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T00:28:40,395 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:28:40,417 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal.1731457720396, exclude list is [], retry=0 2024-11-13T00:28:40,421 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:40,422 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:40,423 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:40,425 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal.1731457720396 2024-11-13T00:28:40,425 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:45019:45019)] 2024-11-13T00:28:40,425 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 687f50a2a27f6c55561cea830ccd0689, NAME => 'testReplayEditsWrittenIntoWAL,,1731457720159.687f50a2a27f6c55561cea830ccd0689.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:28:40,426 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731457720159.687f50a2a27f6c55561cea830ccd0689.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:40,426 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 687f50a2a27f6c55561cea830ccd0689 2024-11-13T00:28:40,426 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 687f50a2a27f6c55561cea830ccd0689 2024-11-13T00:28:40,428 INFO [StoreOpener-687f50a2a27f6c55561cea830ccd0689-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 687f50a2a27f6c55561cea830ccd0689 2024-11-13T00:28:40,430 INFO [StoreOpener-687f50a2a27f6c55561cea830ccd0689-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 687f50a2a27f6c55561cea830ccd0689 columnFamilyName a 2024-11-13T00:28:40,430 DEBUG [StoreOpener-687f50a2a27f6c55561cea830ccd0689-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:40,431 INFO [StoreOpener-687f50a2a27f6c55561cea830ccd0689-1 {}] regionserver.HStore(327): Store=687f50a2a27f6c55561cea830ccd0689/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:40,431 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 687f50a2a27f6c55561cea830ccd0689 2024-11-13T00:28:40,432 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689 2024-11-13T00:28:40,436 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689 2024-11-13T00:28:40,437 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/recovered.edits/0000000000000000002 2024-11-13T00:28:40,446 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/recovered.edits/0000000000000000002 2024-11-13T00:28:40,450 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 687f50a2a27f6c55561cea830ccd0689 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-13T00:28:40,501 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/.tmp/a/9f17b99b45404b5d9e68d80c593cae27 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731457720246/Put/seqid=0 2024-11-13T00:28:40,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741844_1020 (size=5170) 2024-11-13T00:28:40,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741844_1020 (size=5170) 2024-11-13T00:28:40,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741844_1020 (size=5170) 2024-11-13T00:28:40,517 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/.tmp/a/9f17b99b45404b5d9e68d80c593cae27 2024-11-13T00:28:40,561 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/.tmp/a/9f17b99b45404b5d9e68d80c593cae27 as hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/a/9f17b99b45404b5d9e68d80c593cae27 2024-11-13T00:28:40,570 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/a/9f17b99b45404b5d9e68d80c593cae27, entries=2, sequenceid=2, filesize=5.0 K 2024-11-13T00:28:40,576 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 687f50a2a27f6c55561cea830ccd0689 in 125ms, sequenceid=2, compaction requested=false; wal=null 2024-11-13T00:28:40,577 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/recovered.edits/0000000000000000002 2024-11-13T00:28:40,578 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 687f50a2a27f6c55561cea830ccd0689 2024-11-13T00:28:40,578 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 687f50a2a27f6c55561cea830ccd0689 2024-11-13T00:28:40,582 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 687f50a2a27f6c55561cea830ccd0689 2024-11-13T00:28:40,585 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/687f50a2a27f6c55561cea830ccd0689/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-13T00:28:40,586 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 687f50a2a27f6c55561cea830ccd0689; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75349562, jitterRate=0.122795969247818}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T00:28:40,587 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 687f50a2a27f6c55561cea830ccd0689: Writing region info on filesystem at 1731457720426Initializing all the Stores at 1731457720428 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457720428Obtaining lock to block concurrent updates at 1731457720450 (+22 ms)Preparing flush snapshotting stores in 687f50a2a27f6c55561cea830ccd0689 at 1731457720450Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731457720159.687f50a2a27f6c55561cea830ccd0689., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731457720453 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731457720159.687f50a2a27f6c55561cea830ccd0689. at 1731457720453Flushing 687f50a2a27f6c55561cea830ccd0689/a: creating writer at 1731457720454 (+1 ms)Flushing 687f50a2a27f6c55561cea830ccd0689/a: appending metadata at 1731457720492 (+38 ms)Flushing 687f50a2a27f6c55561cea830ccd0689/a: closing flushed file at 1731457720495 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36e3ee3c: reopening flushed file at 1731457720559 (+64 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 687f50a2a27f6c55561cea830ccd0689 in 125ms, sequenceid=2, compaction requested=false; wal=null at 1731457720576 (+17 ms)Cleaning up temporary data from old regions at 1731457720578 (+2 ms)Region opened successfully at 1731457720587 (+9 ms) 2024-11-13T00:28:40,614 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=377 (was 365) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:45704 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:45560 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:33104 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:33222 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:42745/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:33252 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1811965340-172.17.0.2-1731457714216:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1811965340-172.17.0.2-1731457714216:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1811965340-172.17.0.2-1731457714216:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:33190 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=123 (was 116) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1937 (was 1950) 2024-11-13T00:28:40,625 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=377, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=123, ProcessCount=11, AvailableMemoryMB=1937 2024-11-13T00:28:40,641 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:40,643 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:40,644 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T00:28:40,647 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-92581726, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/hregion-92581726, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:28:40,661 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-92581726/hregion-92581726.1731457720648, exclude list is [], retry=0 2024-11-13T00:28:40,665 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:40,665 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:40,666 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:40,669 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-92581726/hregion-92581726.1731457720648 2024-11-13T00:28:40,669 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:41441:41441)] 2024-11-13T00:28:40,670 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ebce8f8c7f1aeffc3fe62ca7302dead9, NAME => 'testReplayEditsWrittenIntoWAL,,1731457720642.ebce8f8c7f1aeffc3fe62ca7302dead9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42745/hbase 2024-11-13T00:28:40,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741846_1022 (size=64) 2024-11-13T00:28:40,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741846_1022 (size=64) 2024-11-13T00:28:40,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741846_1022 (size=64) 2024-11-13T00:28:40,686 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731457720642.ebce8f8c7f1aeffc3fe62ca7302dead9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:40,688 INFO [StoreOpener-ebce8f8c7f1aeffc3fe62ca7302dead9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ebce8f8c7f1aeffc3fe62ca7302dead9 2024-11-13T00:28:40,691 INFO [StoreOpener-ebce8f8c7f1aeffc3fe62ca7302dead9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ebce8f8c7f1aeffc3fe62ca7302dead9 columnFamilyName a 2024-11-13T00:28:40,691 DEBUG [StoreOpener-ebce8f8c7f1aeffc3fe62ca7302dead9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:40,692 INFO [StoreOpener-ebce8f8c7f1aeffc3fe62ca7302dead9-1 {}] regionserver.HStore(327): Store=ebce8f8c7f1aeffc3fe62ca7302dead9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:40,692 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ebce8f8c7f1aeffc3fe62ca7302dead9 2024-11-13T00:28:40,693 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9 2024-11-13T00:28:40,694 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9 2024-11-13T00:28:40,694 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ebce8f8c7f1aeffc3fe62ca7302dead9 2024-11-13T00:28:40,695 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ebce8f8c7f1aeffc3fe62ca7302dead9 2024-11-13T00:28:40,697 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ebce8f8c7f1aeffc3fe62ca7302dead9 2024-11-13T00:28:40,700 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:28:40,700 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ebce8f8c7f1aeffc3fe62ca7302dead9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68336251, jitterRate=0.018289491534233093}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T00:28:40,701 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ebce8f8c7f1aeffc3fe62ca7302dead9: Writing region info on filesystem at 1731457720686Initializing all the Stores at 1731457720688 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457720688Cleaning up temporary data from old regions at 1731457720695 (+7 ms)Region opened successfully at 1731457720700 (+5 ms) 2024-11-13T00:28:40,701 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ebce8f8c7f1aeffc3fe62ca7302dead9, disabling compactions & flushes 2024-11-13T00:28:40,701 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731457720642.ebce8f8c7f1aeffc3fe62ca7302dead9. 2024-11-13T00:28:40,701 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731457720642.ebce8f8c7f1aeffc3fe62ca7302dead9. 2024-11-13T00:28:40,701 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731457720642.ebce8f8c7f1aeffc3fe62ca7302dead9. after waiting 0 ms 2024-11-13T00:28:40,701 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731457720642.ebce8f8c7f1aeffc3fe62ca7302dead9. 2024-11-13T00:28:40,703 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731457720642.ebce8f8c7f1aeffc3fe62ca7302dead9. 2024-11-13T00:28:40,703 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ebce8f8c7f1aeffc3fe62ca7302dead9: Waiting for close lock at 1731457720701Disabling compacts and flushes for region at 1731457720701Disabling writes for close at 1731457720701Writing region close event to WAL at 1731457720702 (+1 ms)Closed at 1731457720702 2024-11-13T00:28:40,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741845_1021 (size=93) 2024-11-13T00:28:40,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741845_1021 (size=93) 2024-11-13T00:28:40,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741845_1021 (size=93) 2024-11-13T00:28:40,711 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T00:28:40,711 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-92581726:(num 1731457720648) 2024-11-13T00:28:40,713 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T00:28:40,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741847_1023 (size=369) 2024-11-13T00:28:40,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741847_1023 (size=369) 2024-11-13T00:28:40,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741847_1023 (size=369) 2024-11-13T00:28:40,726 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T00:28:40,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741848_1024 (size=231) 2024-11-13T00:28:40,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741848_1024 (size=231) 2024-11-13T00:28:40,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741848_1024 (size=231) 2024-11-13T00:28:40,758 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal-2, size=231 (231bytes) 2024-11-13T00:28:40,758 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal-2 2024-11-13T00:28:40,759 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal-2 after 1ms 2024-11-13T00:28:40,763 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal-2 took 4ms 2024-11-13T00:28:40,765 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal-2 so closing down 2024-11-13T00:28:40,765 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T00:28:40,767 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-13T00:28:40,769 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/0000000000000000002-wal-2.temp 2024-11-13T00:28:40,769 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T00:28:40,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741849_1025 (size=231) 2024-11-13T00:28:40,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741849_1025 (size=231) 2024-11-13T00:28:40,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741849_1025 (size=231) 2024-11-13T00:28:40,778 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T00:28:40,781 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/0000000000000000002 2024-11-13T00:28:40,781 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 18 ms; skipped=0; WAL=hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-13T00:28:40,781 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal-2, journal: Splitting hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal-2, size=231 (231bytes) at 1731457720758Finishing writing output for hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal-2 so closing down at 1731457720765 (+7 ms)Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/0000000000000000002-wal-2.temp at 1731457720769 (+4 ms)3 split writer threads finished at 1731457720769Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731457720778 (+9 ms)Rename recovered edits hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/0000000000000000002 at 1731457720781 (+3 ms)Processed 1 edits across 1 Regions in 18 ms; skipped=0; WAL=hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1731457720781 2024-11-13T00:28:40,801 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal-1, size=369 (369bytes) 2024-11-13T00:28:40,801 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal-1 2024-11-13T00:28:40,802 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal-1 after 1ms 2024-11-13T00:28:40,806 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal-1 took 5ms 2024-11-13T00:28:40,809 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal-1 so closing down 2024-11-13T00:28:40,809 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T00:28:40,811 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-13T00:28:40,812 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/0000000000000000001-wal-1.temp 2024-11-13T00:28:40,812 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T00:28:40,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741850_1026 (size=369) 2024-11-13T00:28:40,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741850_1026 (size=369) 2024-11-13T00:28:40,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741850_1026 (size=369) 2024-11-13T00:28:40,822 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T00:28:40,828 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/0000000000000000002, length=231 2024-11-13T00:28:40,830 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/0000000000000000002 2024-11-13T00:28:40,830 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-13T00:28:40,830 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal-1, journal: Splitting hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal-1, size=369 (369bytes) at 1731457720801Finishing writing output for hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal-1 so closing down at 1731457720809 (+8 ms)Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/0000000000000000001-wal-1.temp at 1731457720812 (+3 ms)3 split writer threads finished at 1731457720812Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731457720822 (+10 ms)Rename recovered edits hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/0000000000000000002 at 1731457720830 (+8 ms)Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1731457720830 2024-11-13T00:28:40,831 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T00:28:40,833 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:28:40,847 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal.1731457720834, exclude list is [], retry=0 2024-11-13T00:28:40,852 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:40,852 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:40,853 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:40,855 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal.1731457720834 2024-11-13T00:28:40,856 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:36255:36255)] 2024-11-13T00:28:40,856 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ebce8f8c7f1aeffc3fe62ca7302dead9, NAME => 'testReplayEditsWrittenIntoWAL,,1731457720642.ebce8f8c7f1aeffc3fe62ca7302dead9.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:28:40,856 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731457720642.ebce8f8c7f1aeffc3fe62ca7302dead9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:40,856 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ebce8f8c7f1aeffc3fe62ca7302dead9 2024-11-13T00:28:40,856 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ebce8f8c7f1aeffc3fe62ca7302dead9 2024-11-13T00:28:40,860 INFO [StoreOpener-ebce8f8c7f1aeffc3fe62ca7302dead9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ebce8f8c7f1aeffc3fe62ca7302dead9 2024-11-13T00:28:40,862 INFO [StoreOpener-ebce8f8c7f1aeffc3fe62ca7302dead9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ebce8f8c7f1aeffc3fe62ca7302dead9 columnFamilyName a 2024-11-13T00:28:40,862 DEBUG [StoreOpener-ebce8f8c7f1aeffc3fe62ca7302dead9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:40,863 INFO [StoreOpener-ebce8f8c7f1aeffc3fe62ca7302dead9-1 {}] regionserver.HStore(327): Store=ebce8f8c7f1aeffc3fe62ca7302dead9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:40,863 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ebce8f8c7f1aeffc3fe62ca7302dead9 2024-11-13T00:28:40,864 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9 2024-11-13T00:28:40,866 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9 2024-11-13T00:28:40,868 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/0000000000000000002 2024-11-13T00:28:40,872 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/0000000000000000002 2024-11-13T00:28:40,872 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ebce8f8c7f1aeffc3fe62ca7302dead9 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-13T00:28:40,890 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/.tmp/a/ebe0c55966db43908f6846a8b5f0d431 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731457720711/Put/seqid=0 2024-11-13T00:28:40,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741852_1028 (size=5170) 2024-11-13T00:28:40,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741852_1028 (size=5170) 2024-11-13T00:28:40,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741852_1028 (size=5170) 2024-11-13T00:28:40,900 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/.tmp/a/ebe0c55966db43908f6846a8b5f0d431 2024-11-13T00:28:40,909 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/.tmp/a/ebe0c55966db43908f6846a8b5f0d431 as hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/a/ebe0c55966db43908f6846a8b5f0d431 2024-11-13T00:28:40,918 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/a/ebe0c55966db43908f6846a8b5f0d431, entries=2, sequenceid=2, filesize=5.0 K 2024-11-13T00:28:40,918 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for ebce8f8c7f1aeffc3fe62ca7302dead9 in 46ms, sequenceid=2, compaction requested=false; wal=null 2024-11-13T00:28:40,919 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/0000000000000000002 2024-11-13T00:28:40,919 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ebce8f8c7f1aeffc3fe62ca7302dead9 2024-11-13T00:28:40,919 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ebce8f8c7f1aeffc3fe62ca7302dead9 2024-11-13T00:28:40,921 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ebce8f8c7f1aeffc3fe62ca7302dead9 2024-11-13T00:28:40,924 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/ebce8f8c7f1aeffc3fe62ca7302dead9/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-13T00:28:40,925 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ebce8f8c7f1aeffc3fe62ca7302dead9; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61984024, jitterRate=-0.07636606693267822}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T00:28:40,925 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ebce8f8c7f1aeffc3fe62ca7302dead9: Writing region info on filesystem at 1731457720857Initializing all the Stores at 1731457720858 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457720858Obtaining lock to block concurrent updates at 1731457720872 (+14 ms)Preparing flush snapshotting stores in ebce8f8c7f1aeffc3fe62ca7302dead9 at 1731457720872Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731457720642.ebce8f8c7f1aeffc3fe62ca7302dead9., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731457720872Flushing stores of testReplayEditsWrittenIntoWAL,,1731457720642.ebce8f8c7f1aeffc3fe62ca7302dead9. at 1731457720872Flushing ebce8f8c7f1aeffc3fe62ca7302dead9/a: creating writer at 1731457720873 (+1 ms)Flushing ebce8f8c7f1aeffc3fe62ca7302dead9/a: appending metadata at 1731457720889 (+16 ms)Flushing ebce8f8c7f1aeffc3fe62ca7302dead9/a: closing flushed file at 1731457720889Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a302229: reopening flushed file at 1731457720908 (+19 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for ebce8f8c7f1aeffc3fe62ca7302dead9 in 46ms, sequenceid=2, compaction requested=false; wal=null at 1731457720918 (+10 ms)Cleaning up temporary data from old regions at 1731457720919 (+1 ms)Region opened successfully at 1731457720925 (+6 ms) 2024-11-13T00:28:40,941 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=387 (was 377) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:33104 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:40288 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:33190 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:36004 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:45560 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:59906 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1811965340-172.17.0.2-1731457714216:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1811965340-172.17.0.2-1731457714216:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1811965340-172.17.0.2-1731457714216:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=123 (was 123), ProcessCount=11 (was 11), AvailableMemoryMB=1933 (was 1937) 2024-11-13T00:28:40,953 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=387, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=123, ProcessCount=11, AvailableMemoryMB=1932 2024-11-13T00:28:40,969 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:40,971 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:40,972 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T00:28:40,975 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-34904436, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/hregion-34904436, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:28:40,989 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-34904436/hregion-34904436.1731457720976, exclude list is [], retry=0 2024-11-13T00:28:40,993 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:40,993 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:40,993 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:40,996 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-34904436/hregion-34904436.1731457720976 2024-11-13T00:28:40,996 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:41441:41441)] 2024-11-13T00:28:40,996 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 76046cec3876ca56d918a48dc3d221ee, NAME => 'testReplayEditsWrittenIntoWAL,,1731457720970.76046cec3876ca56d918a48dc3d221ee.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42745/hbase 2024-11-13T00:28:41,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741854_1030 (size=64) 2024-11-13T00:28:41,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741854_1030 (size=64) 2024-11-13T00:28:41,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741854_1030 (size=64) 2024-11-13T00:28:41,009 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731457720970.76046cec3876ca56d918a48dc3d221ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:41,011 INFO [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 76046cec3876ca56d918a48dc3d221ee 2024-11-13T00:28:41,013 INFO [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76046cec3876ca56d918a48dc3d221ee columnFamilyName a 2024-11-13T00:28:41,013 DEBUG [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:41,013 INFO [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] regionserver.HStore(327): Store=76046cec3876ca56d918a48dc3d221ee/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:41,013 INFO [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 76046cec3876ca56d918a48dc3d221ee 2024-11-13T00:28:41,015 INFO [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76046cec3876ca56d918a48dc3d221ee columnFamilyName b 2024-11-13T00:28:41,015 DEBUG [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:41,016 INFO [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] regionserver.HStore(327): Store=76046cec3876ca56d918a48dc3d221ee/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:41,016 INFO [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 76046cec3876ca56d918a48dc3d221ee 2024-11-13T00:28:41,017 INFO [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76046cec3876ca56d918a48dc3d221ee columnFamilyName c 2024-11-13T00:28:41,017 DEBUG [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:41,018 INFO [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] regionserver.HStore(327): Store=76046cec3876ca56d918a48dc3d221ee/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:41,018 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 76046cec3876ca56d918a48dc3d221ee 2024-11-13T00:28:41,019 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee 2024-11-13T00:28:41,020 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee 2024-11-13T00:28:41,021 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 76046cec3876ca56d918a48dc3d221ee 2024-11-13T00:28:41,021 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 76046cec3876ca56d918a48dc3d221ee 2024-11-13T00:28:41,022 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T00:28:41,024 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 76046cec3876ca56d918a48dc3d221ee 2024-11-13T00:28:41,027 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:28:41,028 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 76046cec3876ca56d918a48dc3d221ee; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73107466, jitterRate=0.08938613533973694}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T00:28:41,028 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 76046cec3876ca56d918a48dc3d221ee: Writing region info on filesystem at 1731457721009Initializing all the Stores at 1731457721010 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457721010Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457721011 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457721011Cleaning up temporary data from old regions at 1731457721022 (+11 ms)Region opened successfully at 1731457721028 (+6 ms) 2024-11-13T00:28:41,028 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 76046cec3876ca56d918a48dc3d221ee, disabling compactions & flushes 2024-11-13T00:28:41,028 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731457720970.76046cec3876ca56d918a48dc3d221ee. 2024-11-13T00:28:41,028 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731457720970.76046cec3876ca56d918a48dc3d221ee. 2024-11-13T00:28:41,028 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731457720970.76046cec3876ca56d918a48dc3d221ee. after waiting 0 ms 2024-11-13T00:28:41,028 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731457720970.76046cec3876ca56d918a48dc3d221ee. 2024-11-13T00:28:41,029 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731457720970.76046cec3876ca56d918a48dc3d221ee. 2024-11-13T00:28:41,029 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 76046cec3876ca56d918a48dc3d221ee: Waiting for close lock at 1731457721028Disabling compacts and flushes for region at 1731457721028Disabling writes for close at 1731457721028Writing region close event to WAL at 1731457721029 (+1 ms)Closed at 1731457721029 2024-11-13T00:28:41,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741853_1029 (size=93) 2024-11-13T00:28:41,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741853_1029 (size=93) 2024-11-13T00:28:41,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741853_1029 (size=93) 2024-11-13T00:28:41,037 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T00:28:41,037 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-34904436:(num 1731457720976) 2024-11-13T00:28:41,037 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T00:28:41,039 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:28:41,053 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968/wal.1731457721040, exclude list is [], retry=0 2024-11-13T00:28:41,057 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:41,057 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:41,058 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:41,061 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968/wal.1731457721040 2024-11-13T00:28:41,062 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:45019:45019)] 2024-11-13T00:28:41,289 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968/wal.1731457721040, size=0 (0bytes) 2024-11-13T00:28:41,289 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968/wal.1731457721040 might be still open, length is 0 2024-11-13T00:28:41,289 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968/wal.1731457721040 2024-11-13T00:28:41,290 WARN [IPC Server handler 2 on default port 42745 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968/wal.1731457721040 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-13T00:28:41,290 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968/wal.1731457721040 after 1ms 2024-11-13T00:28:42,122 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:40318 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36087:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40318 dst: /127.0.0.1:36087 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36087 remote=/127.0.0.1:40318]. Total timeout mills is 60000, 59134 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:28:42,123 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:36054 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:40949:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36054 dst: /127.0.0.1:40949 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:28:42,124 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:59950 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41563:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59950 dst: /127.0.0.1:41563 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:28:42,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741855_1032 (size=470675) 2024-11-13T00:28:42,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741855_1032 (size=470675) 2024-11-13T00:28:42,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741855_1032 (size=470675) 2024-11-13T00:28:45,259 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T00:28:45,291 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968/wal.1731457721040 after 4002ms 2024-11-13T00:28:45,297 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968/wal.1731457721040 took 4008ms 2024-11-13T00:28:45,302 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731457721040.temp 2024-11-13T00:28:45,308 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T00:28:45,308 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/recovered.edits/0000000000000000001-wal.1731457721040.temp 2024-11-13T00:28:45,381 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968/wal.1731457721040; continuing. 2024-11-13T00:28:45,381 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968/wal.1731457721040 so closing down 2024-11-13T00:28:45,381 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T00:28:45,382 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T00:28:45,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741856_1033 (size=470683) 2024-11-13T00:28:45,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741856_1033 (size=470683) 2024-11-13T00:28:45,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741856_1033 (size=470683) 2024-11-13T00:28:45,386 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/recovered.edits/0000000000000000001-wal.1731457721040.temp (wrote 3002 edits, skipped 0 edits in 31 ms) 2024-11-13T00:28:45,389 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/recovered.edits/0000000000000000001-wal.1731457721040.temp to hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/recovered.edits/0000000000000003002 2024-11-13T00:28:45,389 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 92 ms; skipped=0; WAL=hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968/wal.1731457721040, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T00:28:45,389 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968/wal.1731457721040, journal: Splitting hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968/wal.1731457721040, size=0 (0bytes) at 1731457721289Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/recovered.edits/0000000000000000001-wal.1731457721040.temp at 1731457725308 (+4019 ms)Split 1024 edits, skipped 0 edits. at 1731457725353 (+45 ms)Split 2048 edits, skipped 0 edits. at 1731457725369 (+16 ms)Finishing writing output for hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968/wal.1731457721040 so closing down at 1731457725381 (+12 ms)3 split writer threads finished at 1731457725382 (+1 ms)Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/recovered.edits/0000000000000000001-wal.1731457721040.temp (wrote 3002 edits, skipped 0 edits in 31 ms) at 1731457725386 (+4 ms)Rename recovered edits hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/recovered.edits/0000000000000000001-wal.1731457721040.temp to hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/recovered.edits/0000000000000003002 at 1731457725389 (+3 ms)Processed 3002 edits across 1 Regions in 92 ms; skipped=0; WAL=hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968/wal.1731457721040, size=0, length=0, corrupted=false, cancelled=false at 1731457725389 2024-11-13T00:28:45,391 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968/wal.1731457721040 to hdfs://localhost:42745/hbase/oldWALs/wal.1731457721040 2024-11-13T00:28:45,392 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/recovered.edits/0000000000000003002 2024-11-13T00:28:45,393 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T00:28:45,395 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:28:45,409 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968/wal.1731457725395, exclude list is [], retry=0 2024-11-13T00:28:45,413 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:45,413 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:45,414 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:45,416 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968/wal.1731457725395 2024-11-13T00:28:45,417 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:36255:36255)] 2024-11-13T00:28:45,417 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731457720970.76046cec3876ca56d918a48dc3d221ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:45,420 INFO [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 76046cec3876ca56d918a48dc3d221ee 2024-11-13T00:28:45,422 INFO [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76046cec3876ca56d918a48dc3d221ee columnFamilyName a 2024-11-13T00:28:45,422 DEBUG [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:45,422 INFO [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] regionserver.HStore(327): Store=76046cec3876ca56d918a48dc3d221ee/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:45,423 INFO [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 76046cec3876ca56d918a48dc3d221ee 2024-11-13T00:28:45,423 INFO [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76046cec3876ca56d918a48dc3d221ee columnFamilyName b 2024-11-13T00:28:45,424 DEBUG [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:45,424 INFO [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] regionserver.HStore(327): Store=76046cec3876ca56d918a48dc3d221ee/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:45,424 INFO [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 76046cec3876ca56d918a48dc3d221ee 2024-11-13T00:28:45,425 INFO [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76046cec3876ca56d918a48dc3d221ee columnFamilyName c 2024-11-13T00:28:45,425 DEBUG [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:45,426 INFO [StoreOpener-76046cec3876ca56d918a48dc3d221ee-1 {}] regionserver.HStore(327): Store=76046cec3876ca56d918a48dc3d221ee/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:45,426 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 76046cec3876ca56d918a48dc3d221ee 2024-11-13T00:28:45,427 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee 2024-11-13T00:28:45,428 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee 2024-11-13T00:28:45,429 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/recovered.edits/0000000000000003002 2024-11-13T00:28:45,459 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T00:28:45,814 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 76046cec3876ca56d918a48dc3d221ee 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T00:28:45,853 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/.tmp/a/571d5fe291f14e00b9a6d7fabbdd6738 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731457721074/Put/seqid=0 2024-11-13T00:28:45,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741858_1035 (size=50463) 2024-11-13T00:28:45,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741858_1035 (size=50463) 2024-11-13T00:28:45,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741858_1035 (size=50463) 2024-11-13T00:28:45,865 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/.tmp/a/571d5fe291f14e00b9a6d7fabbdd6738 2024-11-13T00:28:45,874 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/.tmp/a/571d5fe291f14e00b9a6d7fabbdd6738 as hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/a/571d5fe291f14e00b9a6d7fabbdd6738 2024-11-13T00:28:45,881 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/a/571d5fe291f14e00b9a6d7fabbdd6738, entries=754, sequenceid=754, filesize=49.3 K 2024-11-13T00:28:45,881 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 76046cec3876ca56d918a48dc3d221ee in 67ms, sequenceid=754, compaction requested=false; wal=null 2024-11-13T00:28:45,898 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T00:28:45,898 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 76046cec3876ca56d918a48dc3d221ee 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T00:28:45,909 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/.tmp/a/3bee0f50b5914b2086ebd6f57fa07c7e is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731457721124/Put/seqid=0 2024-11-13T00:28:45,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741859_1036 (size=20072) 2024-11-13T00:28:45,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741859_1036 (size=20072) 2024-11-13T00:28:45,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741859_1036 (size=20072) 2024-11-13T00:28:45,921 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/.tmp/a/3bee0f50b5914b2086ebd6f57fa07c7e 2024-11-13T00:28:45,951 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/.tmp/b/08037f7b9e264105bd5ee9d826bfc6ec is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731457721144/Put/seqid=0 2024-11-13T00:28:45,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741860_1037 (size=35835) 2024-11-13T00:28:45,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741860_1037 (size=35835) 2024-11-13T00:28:45,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741860_1037 (size=35835) 2024-11-13T00:28:45,960 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/.tmp/b/08037f7b9e264105bd5ee9d826bfc6ec 2024-11-13T00:28:45,967 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/.tmp/a/3bee0f50b5914b2086ebd6f57fa07c7e as hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/a/3bee0f50b5914b2086ebd6f57fa07c7e 2024-11-13T00:28:45,973 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/a/3bee0f50b5914b2086ebd6f57fa07c7e, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-13T00:28:45,974 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/.tmp/b/08037f7b9e264105bd5ee9d826bfc6ec as hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/b/08037f7b9e264105bd5ee9d826bfc6ec 2024-11-13T00:28:45,980 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/b/08037f7b9e264105bd5ee9d826bfc6ec, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-13T00:28:45,980 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 76046cec3876ca56d918a48dc3d221ee in 82ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-13T00:28:45,986 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T00:28:45,986 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 76046cec3876ca56d918a48dc3d221ee 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T00:28:45,994 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/.tmp/b/60fbd774b6e6466cb9f4ac8112370fbf is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731457721162/Put/seqid=0 2024-11-13T00:28:46,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741861_1038 (size=35082) 2024-11-13T00:28:46,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741861_1038 (size=35082) 2024-11-13T00:28:46,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741861_1038 (size=35082) 2024-11-13T00:28:46,005 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/.tmp/b/60fbd774b6e6466cb9f4ac8112370fbf 2024-11-13T00:28:46,030 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/.tmp/c/45adbb8f27ab4269820cd2b02c542392 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731457721199/Put/seqid=0 2024-11-13T00:28:46,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741862_1039 (size=20825) 2024-11-13T00:28:46,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741862_1039 (size=20825) 2024-11-13T00:28:46,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741862_1039 (size=20825) 2024-11-13T00:28:46,042 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/.tmp/c/45adbb8f27ab4269820cd2b02c542392 2024-11-13T00:28:46,051 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/.tmp/b/60fbd774b6e6466cb9f4ac8112370fbf as hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/b/60fbd774b6e6466cb9f4ac8112370fbf 2024-11-13T00:28:46,057 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/b/60fbd774b6e6466cb9f4ac8112370fbf, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-13T00:28:46,058 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/.tmp/c/45adbb8f27ab4269820cd2b02c542392 as hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/c/45adbb8f27ab4269820cd2b02c542392 2024-11-13T00:28:46,064 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/c/45adbb8f27ab4269820cd2b02c542392, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-13T00:28:46,065 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 76046cec3876ca56d918a48dc3d221ee in 78ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-13T00:28:46,072 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731457721254/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731457720970.76046cec3876ca56d918a48dc3d221ee. 2024-11-13T00:28:46,078 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/recovered.edits/0000000000000003002 2024-11-13T00:28:46,078 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T00:28:46,078 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 76046cec3876ca56d918a48dc3d221ee 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-13T00:28:46,089 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/.tmp/c/bdf221bf2fe14c0f929fc476cf1a4a46 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731457721206/Put/seqid=0 2024-11-13T00:28:46,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741863_1040 (size=50301) 2024-11-13T00:28:46,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741863_1040 (size=50301) 2024-11-13T00:28:46,108 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/.tmp/c/bdf221bf2fe14c0f929fc476cf1a4a46 2024-11-13T00:28:46,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741863_1040 (size=50301) 2024-11-13T00:28:46,118 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bdf221bf2fe14c0f929fc476cf1a4a46 2024-11-13T00:28:46,120 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/.tmp/c/bdf221bf2fe14c0f929fc476cf1a4a46 as hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/c/bdf221bf2fe14c0f929fc476cf1a4a46 2024-11-13T00:28:46,126 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bdf221bf2fe14c0f929fc476cf1a4a46 2024-11-13T00:28:46,126 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/c/bdf221bf2fe14c0f929fc476cf1a4a46, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-13T00:28:46,126 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 76046cec3876ca56d918a48dc3d221ee in 48ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-13T00:28:46,127 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/recovered.edits/0000000000000003002 2024-11-13T00:28:46,128 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 76046cec3876ca56d918a48dc3d221ee 2024-11-13T00:28:46,129 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 76046cec3876ca56d918a48dc3d221ee 2024-11-13T00:28:46,129 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T00:28:46,131 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 76046cec3876ca56d918a48dc3d221ee 2024-11-13T00:28:46,134 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenIntoWAL/76046cec3876ca56d918a48dc3d221ee/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-13T00:28:46,135 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 76046cec3876ca56d918a48dc3d221ee; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61463636, jitterRate=-0.08412045240402222}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T00:28:46,135 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 76046cec3876ca56d918a48dc3d221ee: Writing region info on filesystem at 1731457725417Initializing all the Stores at 1731457725418 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457725418Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457725420 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457725420Cleaning up temporary data from old regions at 1731457726129 (+709 ms)Region opened successfully at 1731457726135 (+6 ms) 2024-11-13T00:28:46,202 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 76046cec3876ca56d918a48dc3d221ee, disabling compactions & flushes 2024-11-13T00:28:46,202 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731457720970.76046cec3876ca56d918a48dc3d221ee. 2024-11-13T00:28:46,202 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731457720970.76046cec3876ca56d918a48dc3d221ee. 2024-11-13T00:28:46,202 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731457720970.76046cec3876ca56d918a48dc3d221ee. after waiting 0 ms 2024-11-13T00:28:46,203 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731457720970.76046cec3876ca56d918a48dc3d221ee. 2024-11-13T00:28:46,205 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731457720970.76046cec3876ca56d918a48dc3d221ee. 2024-11-13T00:28:46,206 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 76046cec3876ca56d918a48dc3d221ee: Waiting for close lock at 1731457726202Disabling compacts and flushes for region at 1731457726202Disabling writes for close at 1731457726202Writing region close event to WAL at 1731457726205 (+3 ms)Closed at 1731457726205 2024-11-13T00:28:46,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741857_1034 (size=93) 2024-11-13T00:28:46,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741857_1034 (size=93) 2024-11-13T00:28:46,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741857_1034 (size=93) 2024-11-13T00:28:46,214 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T00:28:46,215 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731457725395) 2024-11-13T00:28:46,240 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=403 (was 387) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1334082462_22 at /127.0.0.1:40408 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:35453 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1334082462_22 at /127.0.0.1:60004 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41605 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1334082462_22 at /127.0.0.1:36104 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1334082462_22 at /127.0.0.1:36088 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:42745 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35453 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:41605 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:42745 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=863 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=121 (was 123), ProcessCount=11 (was 11), AvailableMemoryMB=1838 (was 1932) 2024-11-13T00:28:46,255 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=403, OpenFileDescriptor=863, MaxFileDescriptor=1048576, SystemLoadAverage=121, ProcessCount=11, AvailableMemoryMB=1836 2024-11-13T00:28:46,279 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:46,281 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:46,283 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T00:28:46,286 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-06997037, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/hregion-06997037, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:28:46,307 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-06997037/hregion-06997037.1731457726287, exclude list is [], retry=0 2024-11-13T00:28:46,310 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:46,311 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:46,311 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:46,314 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-06997037/hregion-06997037.1731457726287 2024-11-13T00:28:46,315 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:45019:45019)] 2024-11-13T00:28:46,315 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a6967a7840df305d76f1e48c27e807ef, NAME => 'test2727,,1731457726280.a6967a7840df305d76f1e48c27e807ef.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42745/hbase 2024-11-13T00:28:46,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741865_1042 (size=43) 2024-11-13T00:28:46,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741865_1042 (size=43) 2024-11-13T00:28:46,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741865_1042 (size=43) 2024-11-13T00:28:46,327 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731457726280.a6967a7840df305d76f1e48c27e807ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:46,329 INFO [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a6967a7840df305d76f1e48c27e807ef 2024-11-13T00:28:46,331 INFO [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6967a7840df305d76f1e48c27e807ef columnFamilyName a 2024-11-13T00:28:46,331 DEBUG [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:46,332 INFO [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] regionserver.HStore(327): Store=a6967a7840df305d76f1e48c27e807ef/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:46,332 INFO [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a6967a7840df305d76f1e48c27e807ef 2024-11-13T00:28:46,334 INFO [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6967a7840df305d76f1e48c27e807ef columnFamilyName b 2024-11-13T00:28:46,334 DEBUG [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:46,335 INFO [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] regionserver.HStore(327): Store=a6967a7840df305d76f1e48c27e807ef/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:46,335 INFO [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a6967a7840df305d76f1e48c27e807ef 2024-11-13T00:28:46,337 INFO [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6967a7840df305d76f1e48c27e807ef columnFamilyName c 2024-11-13T00:28:46,337 DEBUG [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:46,337 INFO [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] regionserver.HStore(327): Store=a6967a7840df305d76f1e48c27e807ef/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:46,338 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a6967a7840df305d76f1e48c27e807ef 2024-11-13T00:28:46,340 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef 2024-11-13T00:28:46,341 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef 2024-11-13T00:28:46,343 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a6967a7840df305d76f1e48c27e807ef 2024-11-13T00:28:46,343 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a6967a7840df305d76f1e48c27e807ef 2024-11-13T00:28:46,343 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T00:28:46,345 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a6967a7840df305d76f1e48c27e807ef 2024-11-13T00:28:46,349 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:28:46,349 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a6967a7840df305d76f1e48c27e807ef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62100020, jitterRate=-0.07463759183883667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T00:28:46,350 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a6967a7840df305d76f1e48c27e807ef: Writing region info on filesystem at 1731457726327Initializing all the Stores at 1731457726329 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457726329Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457726329Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457726329Cleaning up temporary data from old regions at 1731457726343 (+14 ms)Region opened successfully at 1731457726350 (+7 ms) 2024-11-13T00:28:46,350 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a6967a7840df305d76f1e48c27e807ef, disabling compactions & flushes 2024-11-13T00:28:46,351 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731457726280.a6967a7840df305d76f1e48c27e807ef. 2024-11-13T00:28:46,351 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731457726280.a6967a7840df305d76f1e48c27e807ef. 2024-11-13T00:28:46,351 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731457726280.a6967a7840df305d76f1e48c27e807ef. after waiting 0 ms 2024-11-13T00:28:46,351 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731457726280.a6967a7840df305d76f1e48c27e807ef. 2024-11-13T00:28:46,351 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731457726280.a6967a7840df305d76f1e48c27e807ef. 2024-11-13T00:28:46,351 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a6967a7840df305d76f1e48c27e807ef: Waiting for close lock at 1731457726350Disabling compacts and flushes for region at 1731457726350Disabling writes for close at 1731457726351 (+1 ms)Writing region close event to WAL at 1731457726351Closed at 1731457726351 2024-11-13T00:28:46,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741864_1041 (size=93) 2024-11-13T00:28:46,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741864_1041 (size=93) 2024-11-13T00:28:46,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741864_1041 (size=93) 2024-11-13T00:28:46,360 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T00:28:46,360 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-06997037:(num 1731457726287) 2024-11-13T00:28:46,360 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T00:28:46,362 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:28:46,377 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726363, exclude list is [], retry=0 2024-11-13T00:28:46,380 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:46,380 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:46,381 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:46,383 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726363 2024-11-13T00:28:46,384 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:36255:36255)] 2024-11-13T00:28:46,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741866_1043 (size=344306) 2024-11-13T00:28:46,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741866_1043 (size=344306) 2024-11-13T00:28:46,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741866_1043 (size=344306) 2024-11-13T00:28:46,620 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726363, size=336.2 K (344306bytes) 2024-11-13T00:28:46,620 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726363 2024-11-13T00:28:46,621 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726363 after 1ms 2024-11-13T00:28:46,627 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726363 took 7ms 2024-11-13T00:28:46,632 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731457726363.temp 2024-11-13T00:28:46,648 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000000001-wal.1731457726363.temp 2024-11-13T00:28:46,665 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726363 so closing down 2024-11-13T00:28:46,665 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T00:28:46,667 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T00:28:46,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741867_1044 (size=344306) 2024-11-13T00:28:46,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741867_1044 (size=344306) 2024-11-13T00:28:46,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741867_1044 (size=344306) 2024-11-13T00:28:46,678 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000000001-wal.1731457726363.temp (wrote 3000 edits, skipped 0 edits in 17 ms) 2024-11-13T00:28:46,680 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000000001-wal.1731457726363.temp to hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000003000 2024-11-13T00:28:46,681 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 53 ms; skipped=0; WAL=hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726363, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-11-13T00:28:46,681 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726363, journal: Splitting hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726363, size=336.2 K (344306bytes) at 1731457726620Split 1024 edits, skipped 0 edits. at 1731457726642 (+22 ms)Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000000001-wal.1731457726363.temp at 1731457726649 (+7 ms)Split 2048 edits, skipped 0 edits. at 1731457726653 (+4 ms)Finishing writing output for hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726363 so closing down at 1731457726665 (+12 ms)3 split writer threads finished at 1731457726667 (+2 ms)Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000000001-wal.1731457726363.temp (wrote 3000 edits, skipped 0 edits in 17 ms) at 1731457726678 (+11 ms)Rename recovered edits hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000000001-wal.1731457726363.temp to hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000003000 at 1731457726680 (+2 ms)Processed 3000 edits across 1 Regions in 53 ms; skipped=0; WAL=hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726363, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1731457726681 (+1 ms) 2024-11-13T00:28:46,683 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726363 to hdfs://localhost:42745/hbase/oldWALs/wal.1731457726363 2024-11-13T00:28:46,688 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000003000 2024-11-13T00:28:46,689 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T00:28:46,691 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:28:46,714 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726692, exclude list is [], retry=0 2024-11-13T00:28:46,719 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:46,719 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:46,720 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:46,729 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726692 2024-11-13T00:28:46,729 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:41441:41441)] 2024-11-13T00:28:46,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741868_1045 (size=344433) 2024-11-13T00:28:46,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741868_1045 (size=344433) 2024-11-13T00:28:46,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741868_1045 (size=344433) 2024-11-13T00:28:46,926 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726692, size=336.4 K (344433bytes) 2024-11-13T00:28:46,926 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726692 2024-11-13T00:28:46,926 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726692 after 0ms 2024-11-13T00:28:46,931 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726692 took 6ms 2024-11-13T00:28:46,941 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731457726692.temp 2024-11-13T00:28:46,944 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000003001-wal.1731457726692.temp 2024-11-13T00:28:46,967 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726692 so closing down 2024-11-13T00:28:46,967 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T00:28:46,968 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T00:28:46,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741869_1046 (size=344433) 2024-11-13T00:28:46,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741869_1046 (size=344433) 2024-11-13T00:28:46,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741869_1046 (size=344433) 2024-11-13T00:28:46,988 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000003001-wal.1731457726692.temp (wrote 3000 edits, skipped 0 edits in 11 ms) 2024-11-13T00:28:46,990 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000003001-wal.1731457726692.temp to hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000006000 2024-11-13T00:28:46,991 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 57 ms; skipped=0; WAL=hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726692, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-11-13T00:28:46,991 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726692, journal: Splitting hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726692, size=336.4 K (344433bytes) at 1731457726926Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000003001-wal.1731457726692.temp at 1731457726944 (+18 ms)Split 1024 edits, skipped 0 edits. at 1731457726947 (+3 ms)Split 2048 edits, skipped 0 edits. at 1731457726957 (+10 ms)Finishing writing output for hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726692 so closing down at 1731457726967 (+10 ms)3 split writer threads finished at 1731457726968 (+1 ms)Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000003001-wal.1731457726692.temp (wrote 3000 edits, skipped 0 edits in 11 ms) at 1731457726988 (+20 ms)Rename recovered edits hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000003001-wal.1731457726692.temp to hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000006000 at 1731457726990 (+2 ms)Processed 3000 edits across 1 Regions in 57 ms; skipped=0; WAL=hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726692, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1731457726991 (+1 ms) 2024-11-13T00:28:46,993 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726692 to hdfs://localhost:42745/hbase/oldWALs/wal.1731457726692 2024-11-13T00:28:46,994 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000006000 2024-11-13T00:28:46,994 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T00:28:46,997 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/test2727-manual,16010,1731457726278, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:28:47,013 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726997, exclude list is [], retry=0 2024-11-13T00:28:47,016 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:47,017 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:47,017 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:47,020 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731457726278/wal.1731457726997 2024-11-13T00:28:47,021 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:41441:41441)] 2024-11-13T00:28:47,021 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a6967a7840df305d76f1e48c27e807ef, NAME => 'test2727,,1731457726280.a6967a7840df305d76f1e48c27e807ef.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:28:47,021 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731457726280.a6967a7840df305d76f1e48c27e807ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:47,021 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a6967a7840df305d76f1e48c27e807ef 2024-11-13T00:28:47,021 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a6967a7840df305d76f1e48c27e807ef 2024-11-13T00:28:47,024 INFO [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a6967a7840df305d76f1e48c27e807ef 2024-11-13T00:28:47,025 INFO [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6967a7840df305d76f1e48c27e807ef columnFamilyName a 2024-11-13T00:28:47,025 DEBUG [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:47,025 INFO [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] regionserver.HStore(327): Store=a6967a7840df305d76f1e48c27e807ef/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:47,026 INFO [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a6967a7840df305d76f1e48c27e807ef 2024-11-13T00:28:47,027 INFO [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6967a7840df305d76f1e48c27e807ef columnFamilyName b 2024-11-13T00:28:47,027 DEBUG [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:47,027 INFO [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] regionserver.HStore(327): Store=a6967a7840df305d76f1e48c27e807ef/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:47,028 INFO [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a6967a7840df305d76f1e48c27e807ef 2024-11-13T00:28:47,029 INFO [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6967a7840df305d76f1e48c27e807ef columnFamilyName c 2024-11-13T00:28:47,029 DEBUG [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:47,030 INFO [StoreOpener-a6967a7840df305d76f1e48c27e807ef-1 {}] regionserver.HStore(327): Store=a6967a7840df305d76f1e48c27e807ef/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:47,030 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a6967a7840df305d76f1e48c27e807ef 2024-11-13T00:28:47,031 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef 2024-11-13T00:28:47,033 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef 2024-11-13T00:28:47,034 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000003000 2024-11-13T00:28:47,077 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000003000 2024-11-13T00:28:47,078 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000006000 2024-11-13T00:28:47,115 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000006000 2024-11-13T00:28:47,115 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a6967a7840df305d76f1e48c27e807ef 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-13T00:28:47,137 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/.tmp/a/ce5fa5e3c28a4854997b09a50d02ac67 is 41, key is test2727/a:100/1731457726735/Put/seqid=0 2024-11-13T00:28:47,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741871_1048 (size=84227) 2024-11-13T00:28:47,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741871_1048 (size=84227) 2024-11-13T00:28:47,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741871_1048 (size=84227) 2024-11-13T00:28:47,146 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/.tmp/a/ce5fa5e3c28a4854997b09a50d02ac67 2024-11-13T00:28:47,175 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/.tmp/b/e38c9b2f560f44e59edc5d0b9d74fbce is 41, key is test2727/b:100/1731457726790/Put/seqid=0 2024-11-13T00:28:47,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741872_1049 (size=84609) 2024-11-13T00:28:47,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741872_1049 (size=84609) 2024-11-13T00:28:47,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741872_1049 (size=84609) 2024-11-13T00:28:47,182 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/.tmp/b/e38c9b2f560f44e59edc5d0b9d74fbce 2024-11-13T00:28:47,212 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/.tmp/c/42144745bee445c39ae49e41e6fb34f7 is 41, key is test2727/c:100/1731457726842/Put/seqid=0 2024-11-13T00:28:47,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741873_1050 (size=84609) 2024-11-13T00:28:47,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741873_1050 (size=84609) 2024-11-13T00:28:47,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741873_1050 (size=84609) 2024-11-13T00:28:47,220 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/.tmp/c/42144745bee445c39ae49e41e6fb34f7 2024-11-13T00:28:47,229 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/.tmp/a/ce5fa5e3c28a4854997b09a50d02ac67 as hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/a/ce5fa5e3c28a4854997b09a50d02ac67 2024-11-13T00:28:47,237 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/a/ce5fa5e3c28a4854997b09a50d02ac67, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-13T00:28:47,239 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/.tmp/b/e38c9b2f560f44e59edc5d0b9d74fbce as hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/b/e38c9b2f560f44e59edc5d0b9d74fbce 2024-11-13T00:28:47,246 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/b/e38c9b2f560f44e59edc5d0b9d74fbce, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-13T00:28:47,248 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/.tmp/c/42144745bee445c39ae49e41e6fb34f7 as hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/c/42144745bee445c39ae49e41e6fb34f7 2024-11-13T00:28:47,256 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/c/42144745bee445c39ae49e41e6fb34f7, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-13T00:28:47,257 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for a6967a7840df305d76f1e48c27e807ef in 141ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-13T00:28:47,257 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000003000 2024-11-13T00:28:47,258 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/0000000000000006000 2024-11-13T00:28:47,260 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a6967a7840df305d76f1e48c27e807ef 2024-11-13T00:28:47,260 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a6967a7840df305d76f1e48c27e807ef 2024-11-13T00:28:47,260 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T00:28:47,262 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a6967a7840df305d76f1e48c27e807ef 2024-11-13T00:28:47,265 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/test2727/a6967a7840df305d76f1e48c27e807ef/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-13T00:28:47,266 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a6967a7840df305d76f1e48c27e807ef; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66856289, jitterRate=-0.0037636607885360718}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T00:28:47,267 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a6967a7840df305d76f1e48c27e807ef: Writing region info on filesystem at 1731457727021Initializing all the Stores at 1731457727023 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457727023Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457727023Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457727023Obtaining lock to block concurrent updates at 1731457727115 (+92 ms)Preparing flush snapshotting stores in a6967a7840df305d76f1e48c27e807ef at 1731457727115Finished memstore snapshotting test2727,,1731457726280.a6967a7840df305d76f1e48c27e807ef., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731457727115Flushing stores of test2727,,1731457726280.a6967a7840df305d76f1e48c27e807ef. at 1731457727115Flushing a6967a7840df305d76f1e48c27e807ef/a: creating writer at 1731457727115Flushing a6967a7840df305d76f1e48c27e807ef/a: appending metadata at 1731457727137 (+22 ms)Flushing a6967a7840df305d76f1e48c27e807ef/a: closing flushed file at 1731457727137Flushing a6967a7840df305d76f1e48c27e807ef/b: creating writer at 1731457727153 (+16 ms)Flushing a6967a7840df305d76f1e48c27e807ef/b: appending metadata at 1731457727173 (+20 ms)Flushing a6967a7840df305d76f1e48c27e807ef/b: closing flushed file at 1731457727173Flushing a6967a7840df305d76f1e48c27e807ef/c: creating writer at 1731457727190 (+17 ms)Flushing a6967a7840df305d76f1e48c27e807ef/c: appending metadata at 1731457727211 (+21 ms)Flushing a6967a7840df305d76f1e48c27e807ef/c: closing flushed file at 1731457727211Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f3cd77: reopening flushed file at 1731457727228 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54e76f2: reopening flushed file at 1731457727237 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c9354ab: reopening flushed file at 1731457727247 (+10 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for a6967a7840df305d76f1e48c27e807ef in 141ms, sequenceid=6000, compaction requested=false; wal=null at 1731457727257 (+10 ms)Cleaning up temporary data from old regions at 1731457727260 (+3 ms)Region opened successfully at 1731457727267 (+7 ms) 2024-11-13T00:28:47,268 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-13T00:28:47,268 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a6967a7840df305d76f1e48c27e807ef, disabling compactions & flushes 2024-11-13T00:28:47,268 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731457726280.a6967a7840df305d76f1e48c27e807ef. 2024-11-13T00:28:47,268 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731457726280.a6967a7840df305d76f1e48c27e807ef. 2024-11-13T00:28:47,268 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731457726280.a6967a7840df305d76f1e48c27e807ef. after waiting 0 ms 2024-11-13T00:28:47,268 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731457726280.a6967a7840df305d76f1e48c27e807ef. 2024-11-13T00:28:47,270 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731457726280.a6967a7840df305d76f1e48c27e807ef. 2024-11-13T00:28:47,270 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a6967a7840df305d76f1e48c27e807ef: Waiting for close lock at 1731457727268Disabling compacts and flushes for region at 1731457727268Disabling writes for close at 1731457727268Writing region close event to WAL at 1731457727270 (+2 ms)Closed at 1731457727270 2024-11-13T00:28:47,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741870_1047 (size=93) 2024-11-13T00:28:47,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741870_1047 (size=93) 2024-11-13T00:28:47,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741870_1047 (size=93) 2024-11-13T00:28:47,276 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T00:28:47,277 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731457726997) 2024-11-13T00:28:47,296 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=406 (was 403) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:60134 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:36104 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:36088 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:40460 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=929 (was 863) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=121 (was 121), ProcessCount=11 (was 11), AvailableMemoryMB=1630 (was 1836) 2024-11-13T00:28:47,309 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=406, OpenFileDescriptor=929, MaxFileDescriptor=1048576, SystemLoadAverage=121, ProcessCount=11, AvailableMemoryMB=1627 2024-11-13T00:28:47,326 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:47,334 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731457727325, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:28:47,335 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731457727334 2024-11-13T00:28:47,342 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731457727325/wal.1731457727334 2024-11-13T00:28:47,344 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:36255:36255)] 2024-11-13T00:28:47,351 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 35de3a84fce0289cccc31092fadb9b6e, NAME => 'testSequentialEditLogSeqNum,,1731457727327.35de3a84fce0289cccc31092fadb9b6e.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:28:47,352 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731457727327.35de3a84fce0289cccc31092fadb9b6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:47,352 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 35de3a84fce0289cccc31092fadb9b6e 2024-11-13T00:28:47,352 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 35de3a84fce0289cccc31092fadb9b6e 2024-11-13T00:28:47,353 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:42745/hbase/data/default/testSequentialEditLogSeqNum/35de3a84fce0289cccc31092fadb9b6e doesn't exist for region: 35de3a84fce0289cccc31092fadb9b6e on table testSequentialEditLogSeqNum 2024-11-13T00:28:47,353 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 35de3a84fce0289cccc31092fadb9b6e on table testSequentialEditLogSeqNum 2024-11-13T00:28:47,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741875_1052 (size=62) 2024-11-13T00:28:47,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741875_1052 (size=62) 2024-11-13T00:28:47,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741875_1052 (size=62) 2024-11-13T00:28:47,366 INFO [StoreOpener-35de3a84fce0289cccc31092fadb9b6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 35de3a84fce0289cccc31092fadb9b6e 2024-11-13T00:28:47,368 INFO [StoreOpener-35de3a84fce0289cccc31092fadb9b6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35de3a84fce0289cccc31092fadb9b6e columnFamilyName a 2024-11-13T00:28:47,368 DEBUG [StoreOpener-35de3a84fce0289cccc31092fadb9b6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:47,368 INFO [StoreOpener-35de3a84fce0289cccc31092fadb9b6e-1 {}] regionserver.HStore(327): Store=35de3a84fce0289cccc31092fadb9b6e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:47,368 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 35de3a84fce0289cccc31092fadb9b6e 2024-11-13T00:28:47,369 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testSequentialEditLogSeqNum/35de3a84fce0289cccc31092fadb9b6e 2024-11-13T00:28:47,369 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testSequentialEditLogSeqNum/35de3a84fce0289cccc31092fadb9b6e 2024-11-13T00:28:47,370 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 35de3a84fce0289cccc31092fadb9b6e 2024-11-13T00:28:47,370 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 35de3a84fce0289cccc31092fadb9b6e 2024-11-13T00:28:47,372 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 35de3a84fce0289cccc31092fadb9b6e 2024-11-13T00:28:47,374 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testSequentialEditLogSeqNum/35de3a84fce0289cccc31092fadb9b6e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:28:47,374 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 35de3a84fce0289cccc31092fadb9b6e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60626635, jitterRate=-0.0965927392244339}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T00:28:47,376 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 35de3a84fce0289cccc31092fadb9b6e: Writing region info on filesystem at 1731457727352Initializing all the Stores at 1731457727366 (+14 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457727366Cleaning up temporary data from old regions at 1731457727370 (+4 ms)Region opened successfully at 1731457727376 (+6 ms) 2024-11-13T00:28:47,390 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 35de3a84fce0289cccc31092fadb9b6e 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-13T00:28:47,413 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testSequentialEditLogSeqNum/35de3a84fce0289cccc31092fadb9b6e/.tmp/a/5b6c0d07308241fea3faacfaa8ba3744 is 81, key is testSequentialEditLogSeqNum/a:x0/1731457727376/Put/seqid=0 2024-11-13T00:28:47,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741876_1053 (size=5833) 2024-11-13T00:28:47,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741876_1053 (size=5833) 2024-11-13T00:28:47,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741876_1053 (size=5833) 2024-11-13T00:28:47,421 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testSequentialEditLogSeqNum/35de3a84fce0289cccc31092fadb9b6e/.tmp/a/5b6c0d07308241fea3faacfaa8ba3744 2024-11-13T00:28:47,427 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testSequentialEditLogSeqNum/35de3a84fce0289cccc31092fadb9b6e/.tmp/a/5b6c0d07308241fea3faacfaa8ba3744 as hdfs://localhost:42745/hbase/data/default/testSequentialEditLogSeqNum/35de3a84fce0289cccc31092fadb9b6e/a/5b6c0d07308241fea3faacfaa8ba3744 2024-11-13T00:28:47,433 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testSequentialEditLogSeqNum/35de3a84fce0289cccc31092fadb9b6e/a/5b6c0d07308241fea3faacfaa8ba3744, entries=10, sequenceid=13, filesize=5.7 K 2024-11-13T00:28:47,434 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 35de3a84fce0289cccc31092fadb9b6e in 44ms, sequenceid=13, compaction requested=false 2024-11-13T00:28:47,434 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 35de3a84fce0289cccc31092fadb9b6e: 2024-11-13T00:28:47,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T00:28:47,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T00:28:47,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T00:28:47,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T00:28:47,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T00:28:47,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741874_1051 (size=3017) 2024-11-13T00:28:47,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741874_1051 (size=3017) 2024-11-13T00:28:47,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741874_1051 (size=3017) 2024-11-13T00:28:47,462 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42745/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731457727325/wal.1731457727334, size=2.9 K (3017bytes) 2024-11-13T00:28:47,462 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42745/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731457727325/wal.1731457727334 2024-11-13T00:28:47,463 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42745/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731457727325/wal.1731457727334 after 1ms 2024-11-13T00:28:47,466 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42745/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731457727325/wal.1731457727334 took 4ms 2024-11-13T00:28:47,468 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42745/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731457727325/wal.1731457727334 so closing down 2024-11-13T00:28:47,468 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T00:28:47,470 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731457727334.temp 2024-11-13T00:28:47,471 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testSequentialEditLogSeqNum/35de3a84fce0289cccc31092fadb9b6e/recovered.edits/0000000000000000003-wal.1731457727334.temp 2024-11-13T00:28:47,471 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T00:28:47,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741877_1054 (size=2433) 2024-11-13T00:28:47,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741877_1054 (size=2433) 2024-11-13T00:28:47,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741877_1054 (size=2433) 2024-11-13T00:28:47,484 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testSequentialEditLogSeqNum/35de3a84fce0289cccc31092fadb9b6e/recovered.edits/0000000000000000003-wal.1731457727334.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-13T00:28:47,485 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42745/hbase/data/default/testSequentialEditLogSeqNum/35de3a84fce0289cccc31092fadb9b6e/recovered.edits/0000000000000000003-wal.1731457727334.temp to hdfs://localhost:42745/hbase/data/default/testSequentialEditLogSeqNum/35de3a84fce0289cccc31092fadb9b6e/recovered.edits/0000000000000000020 2024-11-13T00:28:47,486 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 20 ms; skipped=2; WAL=hdfs://localhost:42745/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731457727325/wal.1731457727334, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-11-13T00:28:47,486 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42745/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731457727325/wal.1731457727334, journal: Splitting hdfs://localhost:42745/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731457727325/wal.1731457727334, size=2.9 K (3017bytes) at 1731457727462Finishing writing output for hdfs://localhost:42745/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731457727325/wal.1731457727334 so closing down at 1731457727468 (+6 ms)Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testSequentialEditLogSeqNum/35de3a84fce0289cccc31092fadb9b6e/recovered.edits/0000000000000000003-wal.1731457727334.temp at 1731457727471 (+3 ms)3 split writer threads finished at 1731457727471Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testSequentialEditLogSeqNum/35de3a84fce0289cccc31092fadb9b6e/recovered.edits/0000000000000000003-wal.1731457727334.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731457727484 (+13 ms)Rename recovered edits hdfs://localhost:42745/hbase/data/default/testSequentialEditLogSeqNum/35de3a84fce0289cccc31092fadb9b6e/recovered.edits/0000000000000000003-wal.1731457727334.temp to hdfs://localhost:42745/hbase/data/default/testSequentialEditLogSeqNum/35de3a84fce0289cccc31092fadb9b6e/recovered.edits/0000000000000000020 at 1731457727485 (+1 ms)Processed 17 edits across 1 Regions in 20 ms; skipped=2; WAL=hdfs://localhost:42745/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731457727325/wal.1731457727334, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1731457727486 (+1 ms) 2024-11-13T00:28:47,500 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=411 (was 406) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:60134 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:36104 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:36088 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=967 (was 929) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=121 (was 121), ProcessCount=11 (was 11), AvailableMemoryMB=1613 (was 1627) 2024-11-13T00:28:47,512 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=411, OpenFileDescriptor=967, MaxFileDescriptor=1048576, SystemLoadAverage=121, ProcessCount=11, AvailableMemoryMB=1612 2024-11-13T00:28:47,529 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:47,531 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:47,557 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T00:28:47,560 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-08332022, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/hregion-08332022, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:28:47,576 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-08332022/hregion-08332022.1731457727561, exclude list is [], retry=0 2024-11-13T00:28:47,580 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:47,581 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:47,582 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:47,585 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-08332022/hregion-08332022.1731457727561 2024-11-13T00:28:47,585 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:36255:36255)] 2024-11-13T00:28:47,585 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ab6311c8c7a6c102517b10823d9ea5b3, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731457727529.ab6311c8c7a6c102517b10823d9ea5b3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42745/hbase 2024-11-13T00:28:47,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741879_1056 (size=70) 2024-11-13T00:28:47,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741879_1056 (size=70) 2024-11-13T00:28:47,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741879_1056 (size=70) 2024-11-13T00:28:47,600 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731457727529.ab6311c8c7a6c102517b10823d9ea5b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:47,602 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T00:28:47,602 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:47,602 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T00:28:47,604 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab6311c8c7a6c102517b10823d9ea5b3 columnFamilyName a 2024-11-13T00:28:47,604 DEBUG [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:47,605 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T00:28:47,605 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T00:28:47,605 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] regionserver.HStore(327): Store=ab6311c8c7a6c102517b10823d9ea5b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:47,605 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T00:28:47,605 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:47,605 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T00:28:47,607 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-13T00:28:47,607 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-13T00:28:47,607 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab6311c8c7a6c102517b10823d9ea5b3 columnFamilyName b 2024-11-13T00:28:47,607 DEBUG [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:47,608 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] regionserver.HStore(327): Store=ab6311c8c7a6c102517b10823d9ea5b3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:47,608 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:47,608 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-13T00:28:47,608 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-13T00:28:47,610 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab6311c8c7a6c102517b10823d9ea5b3 columnFamilyName c 2024-11-13T00:28:47,610 DEBUG [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:47,610 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] regionserver.HStore(327): Store=ab6311c8c7a6c102517b10823d9ea5b3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:47,610 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:47,611 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:47,612 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:47,613 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:47,613 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:47,614 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T00:28:47,615 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:47,618 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:28:47,619 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ab6311c8c7a6c102517b10823d9ea5b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61692308, jitterRate=-0.08071297407150269}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T00:28:47,621 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ab6311c8c7a6c102517b10823d9ea5b3: Writing region info on filesystem at 1731457727600Initializing all the Stores at 1731457727601 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457727602 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457727602Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457727602Cleaning up temporary data from old regions at 1731457727613 (+11 ms)Region opened successfully at 1731457727621 (+8 ms) 2024-11-13T00:28:47,621 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ab6311c8c7a6c102517b10823d9ea5b3, disabling compactions & flushes 2024-11-13T00:28:47,621 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731457727529.ab6311c8c7a6c102517b10823d9ea5b3. 2024-11-13T00:28:47,621 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731457727529.ab6311c8c7a6c102517b10823d9ea5b3. 2024-11-13T00:28:47,621 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731457727529.ab6311c8c7a6c102517b10823d9ea5b3. after waiting 0 ms 2024-11-13T00:28:47,621 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731457727529.ab6311c8c7a6c102517b10823d9ea5b3. 2024-11-13T00:28:47,622 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731457727529.ab6311c8c7a6c102517b10823d9ea5b3. 2024-11-13T00:28:47,622 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ab6311c8c7a6c102517b10823d9ea5b3: Waiting for close lock at 1731457727621Disabling compacts and flushes for region at 1731457727621Disabling writes for close at 1731457727621Writing region close event to WAL at 1731457727622 (+1 ms)Closed at 1731457727622 2024-11-13T00:28:47,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741878_1055 (size=93) 2024-11-13T00:28:47,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741878_1055 (size=93) 2024-11-13T00:28:47,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741878_1055 (size=93) 2024-11-13T00:28:47,630 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T00:28:47,630 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-08332022:(num 1731457727561) 2024-11-13T00:28:47,630 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T00:28:47,632 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:28:47,647 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528/wal.1731457727633, exclude list is [], retry=0 2024-11-13T00:28:47,650 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:47,651 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:47,651 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:47,653 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528/wal.1731457727633 2024-11-13T00:28:47,655 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:36255:36255)] 2024-11-13T00:28:47,655 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ab6311c8c7a6c102517b10823d9ea5b3, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731457727529.ab6311c8c7a6c102517b10823d9ea5b3.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:28:47,656 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731457727529.ab6311c8c7a6c102517b10823d9ea5b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:47,656 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:47,656 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:47,660 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:47,662 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab6311c8c7a6c102517b10823d9ea5b3 columnFamilyName a 2024-11-13T00:28:47,662 DEBUG [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:47,662 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] regionserver.HStore(327): Store=ab6311c8c7a6c102517b10823d9ea5b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:47,663 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:47,664 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab6311c8c7a6c102517b10823d9ea5b3 columnFamilyName b 2024-11-13T00:28:47,664 DEBUG [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:47,664 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] regionserver.HStore(327): Store=ab6311c8c7a6c102517b10823d9ea5b3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:47,664 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:47,665 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab6311c8c7a6c102517b10823d9ea5b3 columnFamilyName c 2024-11-13T00:28:47,665 DEBUG [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:47,666 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] regionserver.HStore(327): Store=ab6311c8c7a6c102517b10823d9ea5b3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:47,666 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:47,667 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:47,668 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:47,669 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:47,669 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:47,670 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T00:28:47,671 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:47,672 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ab6311c8c7a6c102517b10823d9ea5b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63750207, jitterRate=-0.05004788935184479}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T00:28:47,673 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ab6311c8c7a6c102517b10823d9ea5b3: Writing region info on filesystem at 1731457727656Initializing all the Stores at 1731457727657 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457727657Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457727660 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457727660Cleaning up temporary data from old regions at 1731457727669 (+9 ms)Region opened successfully at 1731457727673 (+4 ms) 2024-11-13T00:28:47,677 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731457727676/Put/seqid=0 2024-11-13T00:28:47,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741881_1058 (size=4826) 2024-11-13T00:28:47,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741881_1058 (size=4826) 2024-11-13T00:28:47,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741881_1058 (size=4826) 2024-11-13T00:28:47,687 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:42745/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in ab6311c8c7a6c102517b10823d9ea5b3/a 2024-11-13T00:28:47,695 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-13T00:28:47,695 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T00:28:47,695 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ab6311c8c7a6c102517b10823d9ea5b3: 2024-11-13T00:28:47,697 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/a/05a1e24384d24d2ba9ca2a0a60cc519f_SeqId_3_ 2024-11-13T00:28:47,698 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:42745/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into ab6311c8c7a6c102517b10823d9ea5b3/a as hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/a/05a1e24384d24d2ba9ca2a0a60cc519f_SeqId_3_ - updating store file list. 2024-11-13T00:28:47,705 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 05a1e24384d24d2ba9ca2a0a60cc519f_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-13T00:28:47,705 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/a/05a1e24384d24d2ba9ca2a0a60cc519f_SeqId_3_ into ab6311c8c7a6c102517b10823d9ea5b3/a 2024-11-13T00:28:47,705 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:42745/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into ab6311c8c7a6c102517b10823d9ea5b3/a (new location: hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/a/05a1e24384d24d2ba9ca2a0a60cc519f_SeqId_3_) 2024-11-13T00:28:47,754 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42745/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528/wal.1731457727633, size=0 (0bytes) 2024-11-13T00:28:47,754 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:42745/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528/wal.1731457727633 might be still open, length is 0 2024-11-13T00:28:47,754 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42745/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528/wal.1731457727633 2024-11-13T00:28:47,755 WARN [IPC Server handler 1 on default port 42745 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528/wal.1731457727633 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-13T00:28:47,755 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42745/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528/wal.1731457727633 after 1ms 2024-11-13T00:28:48,115 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:40596 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36087:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40596 dst: /127.0.0.1:36087 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36087 remote=/127.0.0.1:40596]. Total timeout mills is 60000, 59597 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:28:48,116 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:36314 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:40949:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36314 dst: /127.0.0.1:40949 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:28:48,116 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:60226 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41563:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60226 dst: /127.0.0.1:41563 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:28:48,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741880_1059 (size=504) 2024-11-13T00:28:48,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741880_1059 (size=504) 2024-11-13T00:28:48,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741880_1059 (size=504) 2024-11-13T00:28:51,756 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42745/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528/wal.1731457727633 after 4002ms 2024-11-13T00:28:51,760 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42745/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528/wal.1731457727633 took 4006ms 2024-11-13T00:28:51,762 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:42745/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528/wal.1731457727633; continuing. 2024-11-13T00:28:51,762 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42745/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528/wal.1731457727633 so closing down 2024-11-13T00:28:51,762 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T00:28:51,764 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731457727633.temp 2024-11-13T00:28:51,765 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/recovered.edits/0000000000000000005-wal.1731457727633.temp 2024-11-13T00:28:51,765 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T00:28:51,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741882_1060 (size=238) 2024-11-13T00:28:51,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741882_1060 (size=238) 2024-11-13T00:28:51,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741882_1060 (size=238) 2024-11-13T00:28:51,774 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/recovered.edits/0000000000000000005-wal.1731457727633.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T00:28:51,775 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/recovered.edits/0000000000000000005-wal.1731457727633.temp to hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/recovered.edits/0000000000000000005 2024-11-13T00:28:51,775 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 15 ms; skipped=1; WAL=hdfs://localhost:42745/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528/wal.1731457727633, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T00:28:51,775 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42745/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528/wal.1731457727633, journal: Splitting hdfs://localhost:42745/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528/wal.1731457727633, size=0 (0bytes) at 1731457727754Finishing writing output for hdfs://localhost:42745/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528/wal.1731457727633 so closing down at 1731457731762 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/recovered.edits/0000000000000000005-wal.1731457727633.temp at 1731457731765 (+3 ms)3 split writer threads finished at 1731457731765Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/recovered.edits/0000000000000000005-wal.1731457727633.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731457731774 (+9 ms)Rename recovered edits hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/recovered.edits/0000000000000000005-wal.1731457727633.temp to hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/recovered.edits/0000000000000000005 at 1731457731775 (+1 ms)Processed 2 edits across 1 Regions in 15 ms; skipped=1; WAL=hdfs://localhost:42745/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528/wal.1731457727633, size=0, length=0, corrupted=false, cancelled=false at 1731457731775 2024-11-13T00:28:51,777 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42745/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528/wal.1731457727633 to hdfs://localhost:42745/hbase/oldWALs/wal.1731457727633 2024-11-13T00:28:51,778 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/recovered.edits/0000000000000000005 2024-11-13T00:28:51,778 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T00:28:51,780 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:28:51,797 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528/wal.1731457731781, exclude list is [], retry=0 2024-11-13T00:28:51,800 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:51,801 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:51,801 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:51,803 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528/wal.1731457731781 2024-11-13T00:28:51,803 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:45019:45019)] 2024-11-13T00:28:51,803 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ab6311c8c7a6c102517b10823d9ea5b3, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731457727529.ab6311c8c7a6c102517b10823d9ea5b3.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:28:51,803 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731457727529.ab6311c8c7a6c102517b10823d9ea5b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:51,803 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:51,804 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:51,805 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:51,806 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab6311c8c7a6c102517b10823d9ea5b3 columnFamilyName a 2024-11-13T00:28:51,806 DEBUG [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:51,812 DEBUG [StoreFileOpener-ab6311c8c7a6c102517b10823d9ea5b3-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 05a1e24384d24d2ba9ca2a0a60cc519f_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-13T00:28:51,813 DEBUG [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/a/05a1e24384d24d2ba9ca2a0a60cc519f_SeqId_3_ 2024-11-13T00:28:51,813 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] regionserver.HStore(327): Store=ab6311c8c7a6c102517b10823d9ea5b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:51,813 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:51,814 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab6311c8c7a6c102517b10823d9ea5b3 columnFamilyName b 2024-11-13T00:28:51,814 DEBUG [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:51,814 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] regionserver.HStore(327): Store=ab6311c8c7a6c102517b10823d9ea5b3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:51,815 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:51,815 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab6311c8c7a6c102517b10823d9ea5b3 columnFamilyName c 2024-11-13T00:28:51,815 DEBUG [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:51,816 INFO [StoreOpener-ab6311c8c7a6c102517b10823d9ea5b3-1 {}] regionserver.HStore(327): Store=ab6311c8c7a6c102517b10823d9ea5b3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:51,816 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:51,817 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:51,818 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:51,819 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/recovered.edits/0000000000000000005 2024-11-13T00:28:51,822 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/recovered.edits/0000000000000000005 2024-11-13T00:28:51,822 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ab6311c8c7a6c102517b10823d9ea5b3 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-13T00:28:51,837 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/.tmp/a/359449aed9be4fbf87cfd03a457aa758 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731457727712/Put/seqid=0 2024-11-13T00:28:51,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741884_1062 (size=5149) 2024-11-13T00:28:51,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741884_1062 (size=5149) 2024-11-13T00:28:51,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741884_1062 (size=5149) 2024-11-13T00:28:51,845 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/.tmp/a/359449aed9be4fbf87cfd03a457aa758 2024-11-13T00:28:51,851 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/.tmp/a/359449aed9be4fbf87cfd03a457aa758 as hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/a/359449aed9be4fbf87cfd03a457aa758 2024-11-13T00:28:51,856 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/a/359449aed9be4fbf87cfd03a457aa758, entries=1, sequenceid=5, filesize=5.0 K 2024-11-13T00:28:51,856 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for ab6311c8c7a6c102517b10823d9ea5b3 in 34ms, sequenceid=5, compaction requested=false; wal=null 2024-11-13T00:28:51,857 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/recovered.edits/0000000000000000005 2024-11-13T00:28:51,858 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:51,858 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:51,859 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T00:28:51,860 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ab6311c8c7a6c102517b10823d9ea5b3 2024-11-13T00:28:51,862 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/ab6311c8c7a6c102517b10823d9ea5b3/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T00:28:51,863 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ab6311c8c7a6c102517b10823d9ea5b3; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66739985, jitterRate=-0.00549672544002533}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T00:28:51,864 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ab6311c8c7a6c102517b10823d9ea5b3: Writing region info on filesystem at 1731457731804Initializing all the Stores at 1731457731805 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457731805Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457731805Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457731805Obtaining lock to block concurrent updates at 1731457731822 (+17 ms)Preparing flush snapshotting stores in ab6311c8c7a6c102517b10823d9ea5b3 at 1731457731822Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731457727529.ab6311c8c7a6c102517b10823d9ea5b3., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731457731822Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731457727529.ab6311c8c7a6c102517b10823d9ea5b3. at 1731457731822Flushing ab6311c8c7a6c102517b10823d9ea5b3/a: creating writer at 1731457731822Flushing ab6311c8c7a6c102517b10823d9ea5b3/a: appending metadata at 1731457731837 (+15 ms)Flushing ab6311c8c7a6c102517b10823d9ea5b3/a: closing flushed file at 1731457731837Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3edbf1a0: reopening flushed file at 1731457731850 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for ab6311c8c7a6c102517b10823d9ea5b3 in 34ms, sequenceid=5, compaction requested=false; wal=null at 1731457731856 (+6 ms)Cleaning up temporary data from old regions at 1731457731858 (+2 ms)Region opened successfully at 1731457731864 (+6 ms) 2024-11-13T00:28:51,867 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ab6311c8c7a6c102517b10823d9ea5b3, disabling compactions & flushes 2024-11-13T00:28:51,868 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731457727529.ab6311c8c7a6c102517b10823d9ea5b3. 2024-11-13T00:28:51,868 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731457727529.ab6311c8c7a6c102517b10823d9ea5b3. 2024-11-13T00:28:51,868 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731457727529.ab6311c8c7a6c102517b10823d9ea5b3. after waiting 0 ms 2024-11-13T00:28:51,868 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731457727529.ab6311c8c7a6c102517b10823d9ea5b3. 2024-11-13T00:28:51,869 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731457727529.ab6311c8c7a6c102517b10823d9ea5b3. 2024-11-13T00:28:51,869 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ab6311c8c7a6c102517b10823d9ea5b3: Waiting for close lock at 1731457731867Disabling compacts and flushes for region at 1731457731867Disabling writes for close at 1731457731868 (+1 ms)Writing region close event to WAL at 1731457731869 (+1 ms)Closed at 1731457731869 2024-11-13T00:28:51,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741883_1061 (size=93) 2024-11-13T00:28:51,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741883_1061 (size=93) 2024-11-13T00:28:51,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741883_1061 (size=93) 2024-11-13T00:28:51,873 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T00:28:51,874 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731457731781) 2024-11-13T00:28:51,888 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=415 (was 411) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:42745 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-432086185_22 at /127.0.0.1:51976 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-432086185_22 at /127.0.0.1:59088 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:42745 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1025 (was 967) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 121) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1484 (was 1612) 2024-11-13T00:28:51,890 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-11-13T00:28:51,902 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=415, OpenFileDescriptor=1025, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=1484 2024-11-13T00:28:51,902 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-11-13T00:28:51,918 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:51,923 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T00:28:51,928 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is cb28580903fb,44487,1731457717196 2024-11-13T00:28:51,930 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@415e15a8 2024-11-13T00:28:51,931 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T00:28:51,933 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T00:28:51,937 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T00:28:51,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-13T00:28:51,945 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T00:28:51,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-13T00:28:51,947 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:51,949 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T00:28:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T00:28:51,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741885_1063 (size=694) 2024-11-13T00:28:51,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741885_1063 (size=694) 2024-11-13T00:28:51,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741885_1063 (size=694) 2024-11-13T00:28:51,961 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1ae5fa43765ef35f2d27709e3e808ead, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d 2024-11-13T00:28:51,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741886_1064 (size=77) 2024-11-13T00:28:51,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741886_1064 (size=77) 2024-11-13T00:28:51,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741886_1064 (size=77) 2024-11-13T00:28:51,970 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:51,970 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 1ae5fa43765ef35f2d27709e3e808ead, disabling compactions & flushes 2024-11-13T00:28:51,970 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:51,970 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:51,970 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. after waiting 0 ms 2024-11-13T00:28:51,970 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:51,971 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:51,971 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1ae5fa43765ef35f2d27709e3e808ead: Waiting for close lock at 1731457731970Disabling compacts and flushes for region at 1731457731970Disabling writes for close at 1731457731970Writing region close event to WAL at 1731457731970Closed at 1731457731970 2024-11-13T00:28:51,972 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T00:28:51,979 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731457731972"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731457731972"}]},"ts":"1731457731972"} 2024-11-13T00:28:51,982 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T00:28:51,984 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T00:28:51,987 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731457731984"}]},"ts":"1731457731984"} 2024-11-13T00:28:51,991 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-13T00:28:51,991 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(204): Hosts are {cb28580903fb=0} racks are {/default-rack=0} 2024-11-13T00:28:51,993 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T00:28:51,993 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T00:28:51,993 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T00:28:51,993 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T00:28:51,993 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T00:28:51,993 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T00:28:51,993 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T00:28:51,993 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T00:28:51,993 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T00:28:51,993 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T00:28:51,994 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, ASSIGN}] 2024-11-13T00:28:51,996 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, ASSIGN 2024-11-13T00:28:51,997 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, ASSIGN; state=OFFLINE, location=cb28580903fb,38771,1731457718028; forceNewPlan=false, retain=false 2024-11-13T00:28:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T00:28:52,149 INFO [cb28580903fb:44487 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T00:28:52,150 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1ae5fa43765ef35f2d27709e3e808ead, regionState=OPENING, regionLocation=cb28580903fb,38771,1731457718028 2024-11-13T00:28:52,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, ASSIGN because future has completed 2024-11-13T00:28:52,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ae5fa43765ef35f2d27709e3e808ead, server=cb28580903fb,38771,1731457718028}] 2024-11-13T00:28:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T00:28:52,308 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T00:28:52,310 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53899, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T00:28:52,315 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:52,316 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1ae5fa43765ef35f2d27709e3e808ead, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:28:52,316 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:52,316 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:52,316 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:52,317 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:52,318 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:52,320 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ae5fa43765ef35f2d27709e3e808ead columnFamilyName cf1 2024-11-13T00:28:52,320 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:52,320 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(327): Store=1ae5fa43765ef35f2d27709e3e808ead/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:52,320 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:52,322 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ae5fa43765ef35f2d27709e3e808ead columnFamilyName cf2 2024-11-13T00:28:52,322 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:52,322 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(327): Store=1ae5fa43765ef35f2d27709e3e808ead/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:52,322 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:52,323 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:52,324 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:52,324 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:52,325 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:52,325 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T00:28:52,327 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:52,329 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:28:52,330 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1ae5fa43765ef35f2d27709e3e808ead; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65738124, jitterRate=-0.020425617694854736}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T00:28:52,330 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:52,331 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1ae5fa43765ef35f2d27709e3e808ead: Running coprocessor pre-open hook at 1731457732317Writing region info on filesystem at 1731457732317Initializing all the Stores at 1731457732318 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457732318Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457732318Cleaning up temporary data from old regions at 1731457732325 (+7 ms)Running coprocessor post-open hooks at 1731457732330 (+5 ms)Region opened successfully at 1731457732331 (+1 ms) 2024-11-13T00:28:52,333 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., pid=6, masterSystemTime=1731457732308 2024-11-13T00:28:52,336 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:52,336 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:52,337 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1ae5fa43765ef35f2d27709e3e808ead, regionState=OPEN, openSeqNum=2, regionLocation=cb28580903fb,38771,1731457718028 2024-11-13T00:28:52,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ae5fa43765ef35f2d27709e3e808ead, server=cb28580903fb,38771,1731457718028 because future has completed 2024-11-13T00:28:52,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T00:28:52,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1ae5fa43765ef35f2d27709e3e808ead, server=cb28580903fb,38771,1731457718028 in 188 msec 2024-11-13T00:28:52,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T00:28:52,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, ASSIGN in 351 msec 2024-11-13T00:28:52,349 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T00:28:52,349 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731457732349"}]},"ts":"1731457732349"} 2024-11-13T00:28:52,352 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-13T00:28:52,354 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T00:28:52,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 415 msec 2024-11-13T00:28:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T00:28:52,577 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-13T00:28:52,577 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-13T00:28:52,579 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T00:28:52,585 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-13T00:28:52,586 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T00:28:52,586 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-13T00:28:52,602 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,38771,1731457718028, seqNum=2] 2024-11-13T00:28:52,604 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T00:28:52,606 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T00:28:52,621 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=1ae5fa43765ef35f2d27709e3e808ead, source=cb28580903fb,38771,1731457718028, destination=cb28580903fb,39009,1731457717889, warming up region on cb28580903fb,39009,1731457717889 2024-11-13T00:28:52,623 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=1ae5fa43765ef35f2d27709e3e808ead, source=cb28580903fb,38771,1731457718028, destination=cb28580903fb,39009,1731457717889, running balancer 2024-11-13T00:28:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, REOPEN/MOVE 2024-11-13T00:28:52,625 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, REOPEN/MOVE 2024-11-13T00:28:52,627 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1ae5fa43765ef35f2d27709e3e808ead, regionState=CLOSING, regionLocation=cb28580903fb,38771,1731457718028 2024-11-13T00:28:52,627 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39009 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39009 {}] regionserver.HRegion(7855): Warmup {ENCODED => 1ae5fa43765ef35f2d27709e3e808ead, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:28:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39009 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:52,628 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:52,629 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, REOPEN/MOVE because future has completed 2024-11-13T00:28:52,630 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ae5fa43765ef35f2d27709e3e808ead columnFamilyName cf1 2024-11-13T00:28:52,630 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T00:28:52,630 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:52,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1ae5fa43765ef35f2d27709e3e808ead, server=cb28580903fb,38771,1731457718028}] 2024-11-13T00:28:52,633 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(327): Store=1ae5fa43765ef35f2d27709e3e808ead/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:52,633 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:52,635 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ae5fa43765ef35f2d27709e3e808ead columnFamilyName cf2 2024-11-13T00:28:52,635 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:52,635 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(327): Store=1ae5fa43765ef35f2d27709e3e808ead/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39009 {}] regionserver.HRegion(1722): Closing 1ae5fa43765ef35f2d27709e3e808ead, disabling compactions & flushes 2024-11-13T00:28:52,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39009 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39009 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39009 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. after waiting 0 ms 2024-11-13T00:28:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39009 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:52,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39009 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39009 {}] regionserver.HRegion(1676): Region close journal for 1ae5fa43765ef35f2d27709e3e808ead: Waiting for close lock at 1731457732636Disabling compacts and flushes for region at 1731457732636Disabling writes for close at 1731457732636Writing region close event to WAL at 1731457732636Closed at 1731457732636 2024-11-13T00:28:52,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-13T00:28:52,787 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:52,787 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T00:28:52,788 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 1ae5fa43765ef35f2d27709e3e808ead, disabling compactions & flushes 2024-11-13T00:28:52,788 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:52,788 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:52,788 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. after waiting 0 ms 2024-11-13T00:28:52,788 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:52,788 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 1ae5fa43765ef35f2d27709e3e808ead 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-13T00:28:52,806 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/.tmp/cf1/1b9619b49afd4d6da633b13f896d5299 is 35, key is r1/cf1:q/1731457732607/Put/seqid=0 2024-11-13T00:28:52,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741887_1065 (size=4783) 2024-11-13T00:28:52,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741887_1065 (size=4783) 2024-11-13T00:28:52,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741887_1065 (size=4783) 2024-11-13T00:28:52,824 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/.tmp/cf1/1b9619b49afd4d6da633b13f896d5299 2024-11-13T00:28:52,833 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/.tmp/cf1/1b9619b49afd4d6da633b13f896d5299 as hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf1/1b9619b49afd4d6da633b13f896d5299 2024-11-13T00:28:52,840 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf1/1b9619b49afd4d6da633b13f896d5299, entries=1, sequenceid=5, filesize=4.7 K 2024-11-13T00:28:52,841 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 1ae5fa43765ef35f2d27709e3e808ead in 53ms, sequenceid=5, compaction requested=false 2024-11-13T00:28:52,841 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-13T00:28:52,847 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T00:28:52,850 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:52,850 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 1ae5fa43765ef35f2d27709e3e808ead: Waiting for close lock at 1731457732788Running coprocessor pre-close hooks at 1731457732788Disabling compacts and flushes for region at 1731457732788Disabling writes for close at 1731457732788Obtaining lock to block concurrent updates at 1731457732788Preparing flush snapshotting stores in 1ae5fa43765ef35f2d27709e3e808ead at 1731457732788Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731457732789 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. at 1731457732789Flushing 1ae5fa43765ef35f2d27709e3e808ead/cf1: creating writer at 1731457732789Flushing 1ae5fa43765ef35f2d27709e3e808ead/cf1: appending metadata at 1731457732806 (+17 ms)Flushing 1ae5fa43765ef35f2d27709e3e808ead/cf1: closing flushed file at 1731457732806Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e2f4da0: reopening flushed file at 1731457732831 (+25 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 1ae5fa43765ef35f2d27709e3e808ead in 53ms, sequenceid=5, compaction requested=false at 1731457732841 (+10 ms)Writing region close event to WAL at 1731457732843 (+2 ms)Running coprocessor post-close hooks at 1731457732847 (+4 ms)Closed at 1731457732850 (+3 ms) 2024-11-13T00:28:52,850 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 1ae5fa43765ef35f2d27709e3e808ead move to cb28580903fb,39009,1731457717889 record at close sequenceid=5 2024-11-13T00:28:52,853 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:52,854 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1ae5fa43765ef35f2d27709e3e808ead, regionState=CLOSED 2024-11-13T00:28:52,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1ae5fa43765ef35f2d27709e3e808ead, server=cb28580903fb,38771,1731457718028 because future has completed 2024-11-13T00:28:52,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T00:28:52,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 1ae5fa43765ef35f2d27709e3e808ead, server=cb28580903fb,38771,1731457718028 in 228 msec 2024-11-13T00:28:52,863 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, REOPEN/MOVE; state=CLOSED, location=cb28580903fb,39009,1731457717889; forceNewPlan=false, retain=false 2024-11-13T00:28:53,013 INFO [cb28580903fb:44487 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T00:28:53,013 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1ae5fa43765ef35f2d27709e3e808ead, regionState=OPENING, regionLocation=cb28580903fb,39009,1731457717889 2024-11-13T00:28:53,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, REOPEN/MOVE because future has completed 2024-11-13T00:28:53,017 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ae5fa43765ef35f2d27709e3e808ead, server=cb28580903fb,39009,1731457717889}] 2024-11-13T00:28:53,174 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:53,174 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1ae5fa43765ef35f2d27709e3e808ead, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:28:53,174 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:53,175 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:53,175 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:53,175 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:53,176 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:53,177 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ae5fa43765ef35f2d27709e3e808ead columnFamilyName cf1 2024-11-13T00:28:53,177 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:53,185 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf1/1b9619b49afd4d6da633b13f896d5299 2024-11-13T00:28:53,185 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(327): Store=1ae5fa43765ef35f2d27709e3e808ead/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:53,185 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:53,186 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ae5fa43765ef35f2d27709e3e808ead columnFamilyName cf2 2024-11-13T00:28:53,187 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:53,187 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(327): Store=1ae5fa43765ef35f2d27709e3e808ead/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:53,187 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:53,188 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:53,190 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:53,191 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:53,191 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:53,192 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T00:28:53,194 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:53,195 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 1ae5fa43765ef35f2d27709e3e808ead; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71008894, jitterRate=0.058114975690841675}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T00:28:53,195 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:53,196 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 1ae5fa43765ef35f2d27709e3e808ead: Running coprocessor pre-open hook at 1731457733175Writing region info on filesystem at 1731457733175Initializing all the Stores at 1731457733176 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457733176Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457733176Cleaning up temporary data from old regions at 1731457733191 (+15 ms)Running coprocessor post-open hooks at 1731457733195 (+4 ms)Region opened successfully at 1731457733195 2024-11-13T00:28:53,197 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., pid=9, masterSystemTime=1731457733169 2024-11-13T00:28:53,199 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:53,199 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:53,200 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1ae5fa43765ef35f2d27709e3e808ead, regionState=OPEN, openSeqNum=9, regionLocation=cb28580903fb,39009,1731457717889 2024-11-13T00:28:53,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ae5fa43765ef35f2d27709e3e808ead, server=cb28580903fb,39009,1731457717889 because future has completed 2024-11-13T00:28:53,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-13T00:28:53,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 1ae5fa43765ef35f2d27709e3e808ead, server=cb28580903fb,39009,1731457717889 in 189 msec 2024-11-13T00:28:53,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, REOPEN/MOVE in 587 msec 2024-11-13T00:28:53,235 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T00:28:53,237 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T00:28:53,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38771 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:48246 deadline: 1731457793241, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=cb28580903fb port=39009 startCode=1731457717889. As of locationSeqNum=5. 2024-11-13T00:28:53,247 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,38771,1731457718028, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,38771,1731457718028, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=cb28580903fb port=39009 startCode=1731457717889. As of locationSeqNum=5. 2024-11-13T00:28:53,248 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,38771,1731457718028, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=cb28580903fb port=39009 startCode=1731457717889. As of locationSeqNum=5. 2024-11-13T00:28:53,248 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,38771,1731457718028, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,39009,1731457717889, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=cb28580903fb port=39009 startCode=1731457717889. As of locationSeqNum=5. 2024-11-13T00:28:53,366 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1ae5fa43765ef35f2d27709e3e808ead 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-13T00:28:53,382 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/.tmp/cf1/4b20077020aa4062b248fe2b42a42841 is 29, key is r1/cf1:/1731457733357/DeleteFamily/seqid=0 2024-11-13T00:28:53,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741888_1066 (size=4906) 2024-11-13T00:28:53,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741888_1066 (size=4906) 2024-11-13T00:28:53,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741888_1066 (size=4906) 2024-11-13T00:28:53,390 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/.tmp/cf1/4b20077020aa4062b248fe2b42a42841 2024-11-13T00:28:53,395 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4b20077020aa4062b248fe2b42a42841 2024-11-13T00:28:53,410 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/.tmp/cf2/02076c8d849d4c6b999160cfb6a3f6d2 is 29, key is r1/cf2:/1731457733357/DeleteFamily/seqid=0 2024-11-13T00:28:53,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741889_1067 (size=4906) 2024-11-13T00:28:53,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741889_1067 (size=4906) 2024-11-13T00:28:53,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741889_1067 (size=4906) 2024-11-13T00:28:53,418 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/.tmp/cf2/02076c8d849d4c6b999160cfb6a3f6d2 2024-11-13T00:28:53,424 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 02076c8d849d4c6b999160cfb6a3f6d2 2024-11-13T00:28:53,425 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/.tmp/cf1/4b20077020aa4062b248fe2b42a42841 as hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf1/4b20077020aa4062b248fe2b42a42841 2024-11-13T00:28:53,431 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4b20077020aa4062b248fe2b42a42841 2024-11-13T00:28:53,431 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf1/4b20077020aa4062b248fe2b42a42841, entries=1, sequenceid=12, filesize=4.8 K 2024-11-13T00:28:53,432 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/.tmp/cf2/02076c8d849d4c6b999160cfb6a3f6d2 as hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf2/02076c8d849d4c6b999160cfb6a3f6d2 2024-11-13T00:28:53,438 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 02076c8d849d4c6b999160cfb6a3f6d2 2024-11-13T00:28:53,438 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf2/02076c8d849d4c6b999160cfb6a3f6d2, entries=1, sequenceid=12, filesize=4.8 K 2024-11-13T00:28:53,439 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 1ae5fa43765ef35f2d27709e3e808ead in 73ms, sequenceid=12, compaction requested=false 2024-11-13T00:28:53,439 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1ae5fa43765ef35f2d27709e3e808ead: 2024-11-13T00:28:53,442 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T00:28:53,443 DEBUG [Time-limited test {}] regionserver.HStore(1541): 1ae5fa43765ef35f2d27709e3e808ead/cf1 is initiating major compaction (all files) 2024-11-13T00:28:53,443 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T00:28:53,443 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:53,444 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 1ae5fa43765ef35f2d27709e3e808ead/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:53,444 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf1/1b9619b49afd4d6da633b13f896d5299, hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf1/4b20077020aa4062b248fe2b42a42841] into tmpdir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/.tmp, totalSize=9.5 K 2024-11-13T00:28:53,445 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1b9619b49afd4d6da633b13f896d5299, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731457732607 2024-11-13T00:28:53,445 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4b20077020aa4062b248fe2b42a42841, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-13T00:28:53,456 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 1ae5fa43765ef35f2d27709e3e808ead#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:28:53,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741890_1068 (size=4626) 2024-11-13T00:28:53,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741890_1068 (size=4626) 2024-11-13T00:28:53,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741890_1068 (size=4626) 2024-11-13T00:28:53,469 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/.tmp/cf1/9189c9f345de40c6830f32e5af933ff6 as hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf1/9189c9f345de40c6830f32e5af933ff6 2024-11-13T00:28:53,481 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 1ae5fa43765ef35f2d27709e3e808ead/cf1 of 1ae5fa43765ef35f2d27709e3e808ead into 9189c9f345de40c6830f32e5af933ff6(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:28:53,481 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 1ae5fa43765ef35f2d27709e3e808ead: 2024-11-13T00:28:53,481 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-13T00:28:53,481 DEBUG [Time-limited test {}] regionserver.HStore(1541): 1ae5fa43765ef35f2d27709e3e808ead/cf2 is initiating major compaction (all files) 2024-11-13T00:28:53,482 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T00:28:53,482 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T00:28:53,482 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 1ae5fa43765ef35f2d27709e3e808ead/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:53,482 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf2/02076c8d849d4c6b999160cfb6a3f6d2] into tmpdir=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/.tmp, totalSize=4.8 K 2024-11-13T00:28:53,482 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 02076c8d849d4c6b999160cfb6a3f6d2, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-13T00:28:53,487 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 1ae5fa43765ef35f2d27709e3e808ead#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T00:28:53,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741891_1069 (size=4592) 2024-11-13T00:28:53,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741891_1069 (size=4592) 2024-11-13T00:28:53,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741891_1069 (size=4592) 2024-11-13T00:28:53,501 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/.tmp/cf2/07d9ab5c62d94224b08d5845ecd45a34 as hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf2/07d9ab5c62d94224b08d5845ecd45a34 2024-11-13T00:28:53,507 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 1ae5fa43765ef35f2d27709e3e808ead/cf2 of 1ae5fa43765ef35f2d27709e3e808ead into 07d9ab5c62d94224b08d5845ecd45a34(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:28:53,507 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 1ae5fa43765ef35f2d27709e3e808ead: 2024-11-13T00:28:53,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=1ae5fa43765ef35f2d27709e3e808ead, source=cb28580903fb,39009,1731457717889, destination=cb28580903fb,38771,1731457718028, warming up region on cb28580903fb,38771,1731457718028 2024-11-13T00:28:53,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=1ae5fa43765ef35f2d27709e3e808ead, source=cb28580903fb,39009,1731457717889, destination=cb28580903fb,38771,1731457718028, running balancer 2024-11-13T00:28:53,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, REOPEN/MOVE 2024-11-13T00:28:53,514 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, REOPEN/MOVE 2024-11-13T00:28:53,516 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1ae5fa43765ef35f2d27709e3e808ead, regionState=CLOSING, regionLocation=cb28580903fb,39009,1731457717889 2024-11-13T00:28:53,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38771 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38771 {}] regionserver.HRegion(7855): Warmup {ENCODED => 1ae5fa43765ef35f2d27709e3e808ead, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:28:53,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38771 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:53,517 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:53,518 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ae5fa43765ef35f2d27709e3e808ead columnFamilyName cf1 2024-11-13T00:28:53,518 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:53,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, REOPEN/MOVE because future has completed 2024-11-13T00:28:53,519 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T00:28:53,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1ae5fa43765ef35f2d27709e3e808ead, server=cb28580903fb,39009,1731457717889}] 2024-11-13T00:28:53,527 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf1/1b9619b49afd4d6da633b13f896d5299 2024-11-13T00:28:53,531 INFO [StoreFileOpener-1ae5fa43765ef35f2d27709e3e808ead-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4b20077020aa4062b248fe2b42a42841 2024-11-13T00:28:53,531 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf1/4b20077020aa4062b248fe2b42a42841 2024-11-13T00:28:53,535 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf1/9189c9f345de40c6830f32e5af933ff6 2024-11-13T00:28:53,536 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(327): Store=1ae5fa43765ef35f2d27709e3e808ead/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:53,536 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:53,537 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ae5fa43765ef35f2d27709e3e808ead columnFamilyName cf2 2024-11-13T00:28:53,537 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:53,542 INFO [StoreFileOpener-1ae5fa43765ef35f2d27709e3e808ead-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 02076c8d849d4c6b999160cfb6a3f6d2 2024-11-13T00:28:53,543 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf2/02076c8d849d4c6b999160cfb6a3f6d2 2024-11-13T00:28:53,546 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf2/07d9ab5c62d94224b08d5845ecd45a34 2024-11-13T00:28:53,547 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(327): Store=1ae5fa43765ef35f2d27709e3e808ead/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38771 {}] regionserver.HRegion(1722): Closing 1ae5fa43765ef35f2d27709e3e808ead, disabling compactions & flushes 2024-11-13T00:28:53,547 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38771 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38771 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38771 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. after waiting 0 ms 2024-11-13T00:28:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38771 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:53,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38771 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:53,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38771 {}] regionserver.HRegion(1676): Region close journal for 1ae5fa43765ef35f2d27709e3e808ead: Waiting for close lock at 1731457733547Disabling compacts and flushes for region at 1731457733547Disabling writes for close at 1731457733547Writing region close event to WAL at 1731457733548 (+1 ms)Closed at 1731457733548 2024-11-13T00:28:53,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-13T00:28:53,673 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:53,673 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T00:28:53,673 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 1ae5fa43765ef35f2d27709e3e808ead, disabling compactions & flushes 2024-11-13T00:28:53,674 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:53,674 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:53,674 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. after waiting 0 ms 2024-11-13T00:28:53,674 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:53,674 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf1/1b9619b49afd4d6da633b13f896d5299, hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf1/4b20077020aa4062b248fe2b42a42841] to archive 2024-11-13T00:28:53,677 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:28:53,681 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf1/1b9619b49afd4d6da633b13f896d5299 to hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf1/1b9619b49afd4d6da633b13f896d5299 2024-11-13T00:28:53,682 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf1/4b20077020aa4062b248fe2b42a42841 to hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf1/4b20077020aa4062b248fe2b42a42841 2024-11-13T00:28:53,695 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf2/02076c8d849d4c6b999160cfb6a3f6d2] to archive 2024-11-13T00:28:53,696 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:28:53,698 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf2/02076c8d849d4c6b999160cfb6a3f6d2 to hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf2/02076c8d849d4c6b999160cfb6a3f6d2 2024-11-13T00:28:53,702 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-13T00:28:53,703 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:53,703 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 1ae5fa43765ef35f2d27709e3e808ead: Waiting for close lock at 1731457733673Running coprocessor pre-close hooks at 1731457733673Disabling compacts and flushes for region at 1731457733673Disabling writes for close at 1731457733674 (+1 ms)Writing region close event to WAL at 1731457733699 (+25 ms)Running coprocessor post-close hooks at 1731457733703 (+4 ms)Closed at 1731457733703 2024-11-13T00:28:53,703 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 1ae5fa43765ef35f2d27709e3e808ead move to cb28580903fb,38771,1731457718028 record at close sequenceid=12 2024-11-13T00:28:53,706 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:53,707 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1ae5fa43765ef35f2d27709e3e808ead, regionState=CLOSED 2024-11-13T00:28:53,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1ae5fa43765ef35f2d27709e3e808ead, server=cb28580903fb,39009,1731457717889 because future has completed 2024-11-13T00:28:53,713 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-13T00:28:53,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 1ae5fa43765ef35f2d27709e3e808ead, server=cb28580903fb,39009,1731457717889 in 192 msec 2024-11-13T00:28:53,715 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, REOPEN/MOVE; state=CLOSED, location=cb28580903fb,38771,1731457718028; forceNewPlan=false, retain=false 2024-11-13T00:28:53,866 INFO [cb28580903fb:44487 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T00:28:53,866 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1ae5fa43765ef35f2d27709e3e808ead, regionState=OPENING, regionLocation=cb28580903fb,38771,1731457718028 2024-11-13T00:28:53,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, REOPEN/MOVE because future has completed 2024-11-13T00:28:53,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ae5fa43765ef35f2d27709e3e808ead, server=cb28580903fb,38771,1731457718028}] 2024-11-13T00:28:54,026 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:54,027 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 1ae5fa43765ef35f2d27709e3e808ead, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:28:54,027 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,027 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:54,027 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,027 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,029 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,030 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ae5fa43765ef35f2d27709e3e808ead columnFamilyName cf1 2024-11-13T00:28:54,030 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:54,036 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf1/9189c9f345de40c6830f32e5af933ff6 2024-11-13T00:28:54,036 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(327): Store=1ae5fa43765ef35f2d27709e3e808ead/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:54,037 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,037 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ae5fa43765ef35f2d27709e3e808ead columnFamilyName cf2 2024-11-13T00:28:54,038 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:54,043 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf2/07d9ab5c62d94224b08d5845ecd45a34 2024-11-13T00:28:54,044 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(327): Store=1ae5fa43765ef35f2d27709e3e808ead/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:54,044 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,044 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,045 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,046 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,046 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,047 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T00:28:54,048 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,049 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 1ae5fa43765ef35f2d27709e3e808ead; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74885838, jitterRate=0.11588594317436218}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T00:28:54,049 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,050 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 1ae5fa43765ef35f2d27709e3e808ead: Running coprocessor pre-open hook at 1731457734028Writing region info on filesystem at 1731457734028Initializing all the Stores at 1731457734029 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457734029Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457734029Cleaning up temporary data from old regions at 1731457734046 (+17 ms)Running coprocessor post-open hooks at 1731457734049 (+3 ms)Region opened successfully at 1731457734050 (+1 ms) 2024-11-13T00:28:54,051 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., pid=12, masterSystemTime=1731457734022 2024-11-13T00:28:54,053 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:54,053 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:54,054 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1ae5fa43765ef35f2d27709e3e808ead, regionState=OPEN, openSeqNum=18, regionLocation=cb28580903fb,38771,1731457718028 2024-11-13T00:28:54,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ae5fa43765ef35f2d27709e3e808ead, server=cb28580903fb,38771,1731457718028 because future has completed 2024-11-13T00:28:54,060 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-13T00:28:54,060 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 1ae5fa43765ef35f2d27709e3e808ead, server=cb28580903fb,38771,1731457718028 in 187 msec 2024-11-13T00:28:54,062 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, REOPEN/MOVE in 548 msec 2024-11-13T00:28:54,066 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T00:28:54,067 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T00:28:54,069 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server cb28580903fb,38771,1731457718028: testing ***** 2024-11-13T00:28:54,069 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-13T00:28:54,071 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-13T00:28:54,072 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-13T00:28:54,074 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-13T00:28:54,075 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-13T00:28:54,081 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 291423760 }, "NonHeapMemoryUsage": { "committed": 170131456, "init": 7667712, "max": -1, "used": 167527600 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "cb28580903fb", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 33, "ProcessCallTime_mean": 7, "ProcessCallTime_25th_percentile": 9, "ProcessCallTime_median": 17, "ProcessCallTime_75th_percentile": 25, "ProcessCallTime_90th_percentile": 29, "ProcessCallTime_95th_percentile": 31, "ProcessCallTime_98th_percentile": 32, "ProcessCallTime_99th_percentile": 32, "ProcessCallTime_99.9th_percentile": 32, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 33, "TotalCallTime_mean": 8, "TotalCallTime_25th_percentile": 9, "TotalCallTime_median": 17, "TotalCallTime_75th_percentile": 25, "TotalCallTime_90th_percentile": 29, "TotalCallTime_95th_percentile": 31, "TotalCallTime_98th_percentile": 32, "TotalCallTime_99th_percentile": 32, "TotalCallTime_99.9th_percentile": 32, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "cb28580903fb", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:49723", "tag.serverName": "cb28580903fb,39009,1731457717889", "tag.clusterId": "039f7df9-b54c-4a07-b722-53d63e7e97fb", "tag.Context": "regionserver", "tag.Hostname": "cb28580903fb", "regionCount": 1, "storeCount": 4, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731457717889, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 0.4, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 199680, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 1, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 4, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 4, "ScanTime_min": 0, "ScanTime_max": 2, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 1, "ScanTime_95th_percentile": 1, "ScanTime_98th_percentile": 1, "ScanTime_99th_percentile": 1, "ScanTime_99.9th_percentile": 1, "ScanTime_TimeRangeCount_0-1": 4, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 2, "Put_min": 2, "Put_max": 10, "Put_mean": 6, "Put_25th_percentile": 4, "Put_median": 6, "Put_75th_percentile": 8, "Put_90th_percentile": 9, "Put_95th_percentile": 9, "Put_98th_percentile": 9, "Put_99th_percentile": 9, "Put_99.9th_percentile": 9, "Put_TimeRangeCount_0-1": 2, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 1, "Get_min": 2, "Get_max": 2, "Get_mean": 2, "Get_25th_percentile": 2, "Get_median": 2, "Get_75th_percentile": 2, "Get_90th_percentile": 2, "Get_95th_percentile": 2, "Get_98th_percentile": 2, "Get_99th_percentile": 2, "Get_99.9th_percentile": 2, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 4, "ScanSize_min": 0, "ScanSize_max": 144, "ScanSize_mean": 72, "ScanSize_25th_percentile": 36, "ScanSize_median": 72, "ScanSize_75th_percentile": 108, "ScanSize_90th_percentile": 129, "ScanSize_95th_percentile": 136, "ScanSize_98th_percentile": 141, "ScanSize_99th_percentile": 142, "ScanSize_99.9th_percentile": 143, "ScanSize_SizeRangeCount_0-10": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-13T00:28:54,084 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44487 {}] master.MasterRpcServices(700): cb28580903fb,38771,1731457718028 reported a fatal error: ***** ABORTING region server cb28580903fb,38771,1731457718028: testing ***** 2024-11-13T00:28:54,086 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cb28580903fb,38771,1731457718028' ***** 2024-11-13T00:28:54,086 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-13T00:28:54,087 INFO [RS:2;cb28580903fb:38771 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T00:28:54,087 INFO [RS:2;cb28580903fb:38771 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-13T00:28:54,087 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T00:28:54,087 INFO [RS:2;cb28580903fb:38771 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-13T00:28:54,087 INFO [RS:2;cb28580903fb:38771 {}] regionserver.HRegionServer(3091): Received CLOSE for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39009 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:45530 deadline: 1731457794087, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=cb28580903fb port=38771 startCode=1731457718028. As of locationSeqNum=12. 2024-11-13T00:28:54,088 INFO [RS:2;cb28580903fb:38771 {}] regionserver.HRegionServer(956): aborting server cb28580903fb,38771,1731457718028 2024-11-13T00:28:54,088 INFO [RS:2;cb28580903fb:38771 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T00:28:54,088 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,39009,1731457717889, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,39009,1731457717889, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=cb28580903fb port=38771 startCode=1731457718028. As of locationSeqNum=12. 2024-11-13T00:28:54,088 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1ae5fa43765ef35f2d27709e3e808ead, disabling compactions & flushes 2024-11-13T00:28:54,088 INFO [RS:2;cb28580903fb:38771 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;cb28580903fb:38771. 2024-11-13T00:28:54,088 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,39009,1731457717889, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=cb28580903fb port=38771 startCode=1731457718028. As of locationSeqNum=12. 2024-11-13T00:28:54,088 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:54,088 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:54,088 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,39009,1731457717889, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,38771,1731457718028, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=cb28580903fb port=38771 startCode=1731457718028. As of locationSeqNum=12. 2024-11-13T00:28:54,088 DEBUG [RS:2;cb28580903fb:38771 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T00:28:54,088 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. after waiting 0 ms 2024-11-13T00:28:54,088 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:54,089 DEBUG [RS:2;cb28580903fb:38771 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:28:54,089 INFO [RS:2;cb28580903fb:38771 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T00:28:54,089 DEBUG [RS:2;cb28580903fb:38771 {}] regionserver.HRegionServer(1325): Online Regions={1ae5fa43765ef35f2d27709e3e808ead=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.} 2024-11-13T00:28:54,089 DEBUG [RS:2;cb28580903fb:38771 {}] regionserver.HRegionServer(1351): Waiting on 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,090 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:54,090 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1ae5fa43765ef35f2d27709e3e808ead: Waiting for close lock at 1731457734088Running coprocessor pre-close hooks at 1731457734088Disabling compacts and flushes for region at 1731457734088Disabling writes for close at 1731457734088Writing region close event to WAL at 1731457734090 (+2 ms)Running coprocessor post-close hooks at 1731457734090Closed at 1731457734090 2024-11-13T00:28:54,090 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:54,123 INFO [regionserver/cb28580903fb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T00:28:54,192 INFO [regionserver/cb28580903fb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T00:28:54,192 INFO [regionserver/cb28580903fb:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T00:28:54,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server cb28580903fb,38771,1731457718028 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:28:54,198 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,38771,1731457718028, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,38771,1731457718028, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server cb28580903fb,38771,1731457718028 aborting 2024-11-13T00:28:54,198 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,38771,1731457718028, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server cb28580903fb,38771,1731457718028 aborting 2024-11-13T00:28:54,198 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,38771,1731457718028, seqNum=12 from cache 2024-11-13T00:28:54,290 INFO [RS:2;cb28580903fb:38771 {}] regionserver.HRegionServer(976): stopping server cb28580903fb,38771,1731457718028; all regions closed. 2024-11-13T00:28:54,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741833_1009 (size=2142) 2024-11-13T00:28:54,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741833_1009 (size=2142) 2024-11-13T00:28:54,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741833_1009 (size=2142) 2024-11-13T00:28:54,294 DEBUG [RS:2;cb28580903fb:38771 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:28:54,294 INFO [RS:2;cb28580903fb:38771 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T00:28:54,294 INFO [RS:2;cb28580903fb:38771 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T00:28:54,295 INFO [RS:2;cb28580903fb:38771 {}] hbase.ChoreService(370): Chore service for: regionserver/cb28580903fb:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T00:28:54,295 INFO [regionserver/cb28580903fb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T00:28:54,295 INFO [RS:2;cb28580903fb:38771 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T00:28:54,295 INFO [RS:2;cb28580903fb:38771 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T00:28:54,295 INFO [RS:2;cb28580903fb:38771 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T00:28:54,295 INFO [RS:2;cb28580903fb:38771 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T00:28:54,295 INFO [RS:2;cb28580903fb:38771 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38771 2024-11-13T00:28:54,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38771-0x1010eb9fb3c0003, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/cb28580903fb,38771,1731457718028 2024-11-13T00:28:54,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T00:28:54,302 INFO [RS:2;cb28580903fb:38771 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T00:28:54,305 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [cb28580903fb,38771,1731457718028] 2024-11-13T00:28:54,306 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/cb28580903fb,38771,1731457718028 already deleted, retry=false 2024-11-13T00:28:54,307 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of cb28580903fb,38771,1731457718028 on cb28580903fb,44487,1731457717196 2024-11-13T00:28:54,311 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure cb28580903fb,38771,1731457718028, splitWal=true, meta=false 2024-11-13T00:28:54,313 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=13 for cb28580903fb,38771,1731457718028 (carryingMeta=false) cb28580903fb,38771,1731457718028/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@73618d5c[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-13T00:28:54,314 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure cb28580903fb,38771,1731457718028, splitWal=true, meta=false 2024-11-13T00:28:54,315 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): cb28580903fb,38771,1731457718028 had 1 regions 2024-11-13T00:28:54,317 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure cb28580903fb,38771,1731457718028, splitWal=true, meta=false, isMeta: false 2024-11-13T00:28:54,318 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38771,1731457718028-splitting 2024-11-13T00:28:54,319 INFO [PEWorker-1 {}] master.SplitWALManager(105): cb28580903fb,38771,1731457718028 WAL count=1, meta=false 2024-11-13T00:28:54,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure cb28580903fb%2C38771%2C1731457718028.1731457719355}] 2024-11-13T00:28:54,327 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=cb28580903fb,38607,1731457717986 2024-11-13T00:28:54,329 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure cb28580903fb%2C38771%2C1731457718028.1731457719355, worker=cb28580903fb,38607,1731457717986}] 2024-11-13T00:28:54,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38771-0x1010eb9fb3c0003, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T00:28:54,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38771-0x1010eb9fb3c0003, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T00:28:54,405 INFO [RS:2;cb28580903fb:38771 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T00:28:54,406 INFO [RS:2;cb28580903fb:38771 {}] regionserver.HRegionServer(1031): Exiting; stopping=cb28580903fb,38771,1731457718028; zookeeper connection closed. 2024-11-13T00:28:54,406 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@263a2b64 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@263a2b64 2024-11-13T00:28:54,411 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,38771,1731457718028, seqNum=18] 2024-11-13T00:28:54,412 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server cb28580903fb:38771 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: cb28580903fb/172.17.0.2:38771 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:28:54,413 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,38771,1731457718028, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,38771,1731457718028, seqNum=18, error=java.net.ConnectException: Call to address=cb28580903fb:38771 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: cb28580903fb/172.17.0.2:38771 2024-11-13T00:28:54,413 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,38771,1731457718028, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-13T00:28:54,413 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,38771,1731457718028, seqNum=18 from cache 2024-11-13T00:28:54,414 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address cb28580903fb:38771 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: cb28580903fb/172.17.0.2:38771 2024-11-13T00:28:54,491 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T00:28:54,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48997, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T00:28:54,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38607 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-13T00:28:54,513 INFO [RS_LOG_REPLAY_OPS-regionserver/cb28580903fb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38771,1731457718028-splitting/cb28580903fb%2C38771%2C1731457718028.1731457719355, size=2.1 K (2142bytes) 2024-11-13T00:28:54,513 INFO [RS_LOG_REPLAY_OPS-regionserver/cb28580903fb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38771,1731457718028-splitting/cb28580903fb%2C38771%2C1731457718028.1731457719355 2024-11-13T00:28:54,513 INFO [RS_LOG_REPLAY_OPS-regionserver/cb28580903fb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38771,1731457718028-splitting/cb28580903fb%2C38771%2C1731457718028.1731457719355 after 0ms 2024-11-13T00:28:54,517 INFO [RS_LOG_REPLAY_OPS-regionserver/cb28580903fb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38771,1731457718028-splitting/cb28580903fb%2C38771%2C1731457718028.1731457719355 took 5ms 2024-11-13T00:28:54,526 DEBUG [RS_LOG_REPLAY_OPS-regionserver/cb28580903fb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1ae5fa43765ef35f2d27709e3e808ead: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-13T00:28:54,526 DEBUG [RS_LOG_REPLAY_OPS-regionserver/cb28580903fb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38771,1731457718028-splitting/cb28580903fb%2C38771%2C1731457718028.1731457719355 so closing down 2024-11-13T00:28:54,526 DEBUG [RS_LOG_REPLAY_OPS-regionserver/cb28580903fb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T00:28:54,527 INFO [RS_LOG_REPLAY_OPS-regionserver/cb28580903fb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T00:28:54,527 INFO [RS_LOG_REPLAY_OPS-regionserver/cb28580903fb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 10 ms; skipped=6; WAL=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38771,1731457718028-splitting/cb28580903fb%2C38771%2C1731457718028.1731457719355, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-11-13T00:28:54,527 DEBUG [RS_LOG_REPLAY_OPS-regionserver/cb28580903fb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38771,1731457718028-splitting/cb28580903fb%2C38771%2C1731457718028.1731457719355, journal: Splitting hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38771,1731457718028-splitting/cb28580903fb%2C38771%2C1731457718028.1731457719355, size=2.1 K (2142bytes) at 1731457734513Finishing writing output for hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38771,1731457718028-splitting/cb28580903fb%2C38771%2C1731457718028.1731457719355 so closing down at 1731457734526 (+13 ms)3 split writer threads finished at 1731457734527 (+1 ms)Processed 6 edits across 0 Regions in 10 ms; skipped=6; WAL=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38771,1731457718028-splitting/cb28580903fb%2C38771%2C1731457718028.1731457719355, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1731457734527 2024-11-13T00:28:54,527 DEBUG [RS_LOG_REPLAY_OPS-regionserver/cb28580903fb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38771,1731457718028-splitting/cb28580903fb%2C38771%2C1731457718028.1731457719355 2024-11-13T00:28:54,529 DEBUG [RS_LOG_REPLAY_OPS-regionserver/cb28580903fb:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-13T00:28:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44487 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-13T00:28:54,538 INFO [PEWorker-3 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38771,1731457718028-splitting/cb28580903fb%2C38771%2C1731457718028.1731457719355 to hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/oldWALs 2024-11-13T00:28:54,541 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-13T00:28:54,541 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure cb28580903fb%2C38771%2C1731457718028.1731457719355, worker=cb28580903fb,38607,1731457717986 in 209 msec 2024-11-13T00:28:54,542 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=cb28580903fb,38607,1731457717986 2024-11-13T00:28:54,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T00:28:54,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure cb28580903fb%2C38771%2C1731457718028.1731457719355, worker=cb28580903fb,38607,1731457717986 in 222 msec 2024-11-13T00:28:54,548 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38771,1731457718028-splitting dir is empty, no logs to split. 2024-11-13T00:28:54,548 INFO [PEWorker-1 {}] master.SplitWALManager(105): cb28580903fb,38771,1731457718028 WAL count=0, meta=false 2024-11-13T00:28:54,548 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if cb28580903fb,38771,1731457718028 WAL splitting is done? wals=0, meta=false 2024-11-13T00:28:54,550 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for cb28580903fb,38771,1731457718028 failed, ignore...File hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/WALs/cb28580903fb,38771,1731457718028-splitting does not exist. 2024-11-13T00:28:54,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, ASSIGN}] 2024-11-13T00:28:54,554 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, ASSIGN 2024-11-13T00:28:54,555 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T00:28:54,706 DEBUG [cb28580903fb:44487 {}] balancer.BalancerClusterState(204): Hosts are {cb28580903fb=0} racks are {/default-rack=0} 2024-11-13T00:28:54,706 DEBUG [cb28580903fb:44487 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T00:28:54,706 DEBUG [cb28580903fb:44487 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T00:28:54,706 DEBUG [cb28580903fb:44487 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T00:28:54,706 DEBUG [cb28580903fb:44487 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T00:28:54,706 INFO [cb28580903fb:44487 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T00:28:54,706 INFO [cb28580903fb:44487 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T00:28:54,706 DEBUG [cb28580903fb:44487 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T00:28:54,707 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1ae5fa43765ef35f2d27709e3e808ead, regionState=OPENING, regionLocation=cb28580903fb,39009,1731457717889 2024-11-13T00:28:54,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, ASSIGN because future has completed 2024-11-13T00:28:54,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ae5fa43765ef35f2d27709e3e808ead, server=cb28580903fb,39009,1731457717889}] 2024-11-13T00:28:54,730 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,38771,1731457718028, seqNum=18] 2024-11-13T00:28:54,731 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to cb28580903fb:38771 this server is in the failed servers list 2024-11-13T00:28:54,732 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,38771,1731457718028, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,38771,1731457718028, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=cb28580903fb:38771 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: cb28580903fb:38771 2024-11-13T00:28:54,732 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,38771,1731457718028, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: cb28580903fb:38771 2024-11-13T00:28:54,732 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,38771,1731457718028, seqNum=18 from cache 2024-11-13T00:28:54,867 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:54,867 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1ae5fa43765ef35f2d27709e3e808ead, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:28:54,867 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,868 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:54,868 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,868 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,869 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,870 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ae5fa43765ef35f2d27709e3e808ead columnFamilyName cf1 2024-11-13T00:28:54,870 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:54,876 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf1/9189c9f345de40c6830f32e5af933ff6 2024-11-13T00:28:54,876 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(327): Store=1ae5fa43765ef35f2d27709e3e808ead/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:54,876 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,877 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ae5fa43765ef35f2d27709e3e808ead columnFamilyName cf2 2024-11-13T00:28:54,877 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:54,883 DEBUG [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/cf2/07d9ab5c62d94224b08d5845ecd45a34 2024-11-13T00:28:54,883 INFO [StoreOpener-1ae5fa43765ef35f2d27709e3e808ead-1 {}] regionserver.HStore(327): Store=1ae5fa43765ef35f2d27709e3e808ead/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:54,883 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,884 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,885 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,885 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,885 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,886 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T00:28:54,887 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,888 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 1ae5fa43765ef35f2d27709e3e808ead; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61726634, jitterRate=-0.08020147681236267}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T00:28:54,888 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:28:54,889 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 1ae5fa43765ef35f2d27709e3e808ead: Running coprocessor pre-open hook at 1731457734868Writing region info on filesystem at 1731457734868Initializing all the Stores at 1731457734869 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457734869Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457734869Cleaning up temporary data from old regions at 1731457734885 (+16 ms)Running coprocessor post-open hooks at 1731457734888 (+3 ms)Region opened successfully at 1731457734889 (+1 ms) 2024-11-13T00:28:54,890 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., pid=17, masterSystemTime=1731457734863 2024-11-13T00:28:54,892 DEBUG [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:54,892 INFO [RS_OPEN_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:28:54,893 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1ae5fa43765ef35f2d27709e3e808ead, regionState=OPEN, openSeqNum=18, regionLocation=cb28580903fb,39009,1731457717889 2024-11-13T00:28:54,895 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ae5fa43765ef35f2d27709e3e808ead, server=cb28580903fb,39009,1731457717889 because future has completed 2024-11-13T00:28:54,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-13T00:28:54,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1ae5fa43765ef35f2d27709e3e808ead, server=cb28580903fb,39009,1731457717889 in 186 msec 2024-11-13T00:28:54,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-13T00:28:54,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1ae5fa43765ef35f2d27709e3e808ead, ASSIGN in 345 msec 2024-11-13T00:28:54,900 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(291): removed crashed server cb28580903fb,38771,1731457718028 after splitting done 2024-11-13T00:28:54,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure cb28580903fb,38771,1731457718028, splitWal=true, meta=false in 593 msec 2024-11-13T00:28:55,239 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead., hostname=cb28580903fb,39009,1731457717889, seqNum=18] 2024-11-13T00:28:55,254 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=410 (was 415), OpenFileDescriptor=1051 (was 1025) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 127), ProcessCount=11 (was 11), AvailableMemoryMB=1479 (was 1484) 2024-11-13T00:28:55,254 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-11-13T00:28:55,266 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=410, OpenFileDescriptor=1051, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=1478 2024-11-13T00:28:55,266 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-11-13T00:28:55,282 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:55,284 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:55,285 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T00:28:55,287 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-98984397, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/hregion-98984397, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:28:55,300 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-98984397/hregion-98984397.1731457735287, exclude list is [], retry=0 2024-11-13T00:28:55,302 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:55,302 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:55,303 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:55,305 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-98984397/hregion-98984397.1731457735287 2024-11-13T00:28:55,305 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:36255:36255)] 2024-11-13T00:28:55,305 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => aa9a2b52c7d82fdc4527eaa78c0c5ccf, NAME => 'testReplayEditsWrittenViaHRegion,,1731457735283.aa9a2b52c7d82fdc4527eaa78c0c5ccf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42745/hbase 2024-11-13T00:28:55,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741893_1071 (size=67) 2024-11-13T00:28:55,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741893_1071 (size=67) 2024-11-13T00:28:55,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741893_1071 (size=67) 2024-11-13T00:28:55,314 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731457735283.aa9a2b52c7d82fdc4527eaa78c0c5ccf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:55,316 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,317 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa9a2b52c7d82fdc4527eaa78c0c5ccf columnFamilyName a 2024-11-13T00:28:55,317 DEBUG [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:55,319 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] regionserver.HStore(327): Store=aa9a2b52c7d82fdc4527eaa78c0c5ccf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:55,319 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,320 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa9a2b52c7d82fdc4527eaa78c0c5ccf columnFamilyName b 2024-11-13T00:28:55,320 DEBUG [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:55,321 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] regionserver.HStore(327): Store=aa9a2b52c7d82fdc4527eaa78c0c5ccf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:55,321 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,322 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa9a2b52c7d82fdc4527eaa78c0c5ccf columnFamilyName c 2024-11-13T00:28:55,322 DEBUG [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:55,322 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] regionserver.HStore(327): Store=aa9a2b52c7d82fdc4527eaa78c0c5ccf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:55,322 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,323 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,323 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,324 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,324 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,325 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T00:28:55,326 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,328 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:28:55,328 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened aa9a2b52c7d82fdc4527eaa78c0c5ccf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66339371, jitterRate=-0.011466339230537415}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T00:28:55,329 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for aa9a2b52c7d82fdc4527eaa78c0c5ccf: Writing region info on filesystem at 1731457735314Initializing all the Stores at 1731457735315 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457735315Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457735315Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457735315Cleaning up temporary data from old regions at 1731457735324 (+9 ms)Region opened successfully at 1731457735329 (+5 ms) 2024-11-13T00:28:55,329 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing aa9a2b52c7d82fdc4527eaa78c0c5ccf, disabling compactions & flushes 2024-11-13T00:28:55,329 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731457735283.aa9a2b52c7d82fdc4527eaa78c0c5ccf. 2024-11-13T00:28:55,329 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731457735283.aa9a2b52c7d82fdc4527eaa78c0c5ccf. 2024-11-13T00:28:55,329 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731457735283.aa9a2b52c7d82fdc4527eaa78c0c5ccf. after waiting 0 ms 2024-11-13T00:28:55,329 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731457735283.aa9a2b52c7d82fdc4527eaa78c0c5ccf. 2024-11-13T00:28:55,330 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731457735283.aa9a2b52c7d82fdc4527eaa78c0c5ccf. 2024-11-13T00:28:55,330 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for aa9a2b52c7d82fdc4527eaa78c0c5ccf: Waiting for close lock at 1731457735329Disabling compacts and flushes for region at 1731457735329Disabling writes for close at 1731457735329Writing region close event to WAL at 1731457735330 (+1 ms)Closed at 1731457735330 2024-11-13T00:28:55,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741892_1070 (size=93) 2024-11-13T00:28:55,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741892_1070 (size=93) 2024-11-13T00:28:55,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741892_1070 (size=93) 2024-11-13T00:28:55,335 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T00:28:55,335 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-98984397:(num 1731457735287) 2024-11-13T00:28:55,335 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T00:28:55,337 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731457735281, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:28:55,350 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731457735281/wal.1731457735337, exclude list is [], retry=0 2024-11-13T00:28:55,353 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:55,353 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:55,354 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:55,355 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731457735281/wal.1731457735337 2024-11-13T00:28:55,356 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:45019:45019)] 2024-11-13T00:28:55,356 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => aa9a2b52c7d82fdc4527eaa78c0c5ccf, NAME => 'testReplayEditsWrittenViaHRegion,,1731457735283.aa9a2b52c7d82fdc4527eaa78c0c5ccf.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:28:55,356 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731457735283.aa9a2b52c7d82fdc4527eaa78c0c5ccf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:55,356 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,356 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,357 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,358 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa9a2b52c7d82fdc4527eaa78c0c5ccf columnFamilyName a 2024-11-13T00:28:55,358 DEBUG [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:55,359 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] regionserver.HStore(327): Store=aa9a2b52c7d82fdc4527eaa78c0c5ccf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:55,359 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,360 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa9a2b52c7d82fdc4527eaa78c0c5ccf columnFamilyName b 2024-11-13T00:28:55,360 DEBUG [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:55,360 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] regionserver.HStore(327): Store=aa9a2b52c7d82fdc4527eaa78c0c5ccf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:55,360 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,361 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa9a2b52c7d82fdc4527eaa78c0c5ccf columnFamilyName c 2024-11-13T00:28:55,361 DEBUG [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:55,361 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] regionserver.HStore(327): Store=aa9a2b52c7d82fdc4527eaa78c0c5ccf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:55,362 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,362 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,363 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,364 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,364 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,365 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T00:28:55,366 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,367 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened aa9a2b52c7d82fdc4527eaa78c0c5ccf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71929904, jitterRate=0.0718390941619873}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T00:28:55,367 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for aa9a2b52c7d82fdc4527eaa78c0c5ccf: Writing region info on filesystem at 1731457735356Initializing all the Stores at 1731457735357 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457735357Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457735357Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457735357Cleaning up temporary data from old regions at 1731457735364 (+7 ms)Region opened successfully at 1731457735367 (+3 ms) 2024-11-13T00:28:55,391 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing aa9a2b52c7d82fdc4527eaa78c0c5ccf 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-13T00:28:55,407 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/.tmp/a/86f962db221c44a0b64c840903391382 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731457735367/Put/seqid=0 2024-11-13T00:28:55,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741895_1073 (size=5958) 2024-11-13T00:28:55,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741895_1073 (size=5958) 2024-11-13T00:28:55,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741895_1073 (size=5958) 2024-11-13T00:28:55,414 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/.tmp/a/86f962db221c44a0b64c840903391382 2024-11-13T00:28:55,441 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/.tmp/b/db369fa70bdc4d2ab61f695b83b04e75 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731457735375/Put/seqid=0 2024-11-13T00:28:55,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741896_1074 (size=5958) 2024-11-13T00:28:55,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741896_1074 (size=5958) 2024-11-13T00:28:55,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741896_1074 (size=5958) 2024-11-13T00:28:55,450 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/.tmp/b/db369fa70bdc4d2ab61f695b83b04e75 2024-11-13T00:28:55,477 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/.tmp/c/a8466e7e38e84e0fa7b47049019ca58d is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731457735382/Put/seqid=0 2024-11-13T00:28:55,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741897_1075 (size=5958) 2024-11-13T00:28:55,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741897_1075 (size=5958) 2024-11-13T00:28:55,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741897_1075 (size=5958) 2024-11-13T00:28:55,488 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/.tmp/c/a8466e7e38e84e0fa7b47049019ca58d 2024-11-13T00:28:55,495 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/.tmp/a/86f962db221c44a0b64c840903391382 as hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/a/86f962db221c44a0b64c840903391382 2024-11-13T00:28:55,501 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/a/86f962db221c44a0b64c840903391382, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T00:28:55,502 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/.tmp/b/db369fa70bdc4d2ab61f695b83b04e75 as hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/b/db369fa70bdc4d2ab61f695b83b04e75 2024-11-13T00:28:55,508 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/b/db369fa70bdc4d2ab61f695b83b04e75, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T00:28:55,509 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/.tmp/c/a8466e7e38e84e0fa7b47049019ca58d as hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/c/a8466e7e38e84e0fa7b47049019ca58d 2024-11-13T00:28:55,514 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/c/a8466e7e38e84e0fa7b47049019ca58d, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T00:28:55,516 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for aa9a2b52c7d82fdc4527eaa78c0c5ccf in 125ms, sequenceid=33, compaction requested=false 2024-11-13T00:28:55,516 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for aa9a2b52c7d82fdc4527eaa78c0c5ccf: 2024-11-13T00:28:55,516 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing aa9a2b52c7d82fdc4527eaa78c0c5ccf, disabling compactions & flushes 2024-11-13T00:28:55,516 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731457735283.aa9a2b52c7d82fdc4527eaa78c0c5ccf. 2024-11-13T00:28:55,516 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731457735283.aa9a2b52c7d82fdc4527eaa78c0c5ccf. 2024-11-13T00:28:55,516 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731457735283.aa9a2b52c7d82fdc4527eaa78c0c5ccf. after waiting 0 ms 2024-11-13T00:28:55,516 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731457735283.aa9a2b52c7d82fdc4527eaa78c0c5ccf. 2024-11-13T00:28:55,518 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731457735283.aa9a2b52c7d82fdc4527eaa78c0c5ccf. 2024-11-13T00:28:55,518 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for aa9a2b52c7d82fdc4527eaa78c0c5ccf: Waiting for close lock at 1731457735516Disabling compacts and flushes for region at 1731457735516Disabling writes for close at 1731457735516Writing region close event to WAL at 1731457735518 (+2 ms)Closed at 1731457735518 2024-11-13T00:28:55,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741894_1072 (size=5937) 2024-11-13T00:28:55,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741894_1072 (size=5937) 2024-11-13T00:28:55,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741894_1072 (size=5937) 2024-11-13T00:28:55,526 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/b/db369fa70bdc4d2ab61f695b83b04e75 to hdfs://localhost:42745/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/b/db369fa70bdc4d2ab61f695b83b04e75 2024-11-13T00:28:55,544 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42745/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731457735281/wal.1731457735337, size=5.8 K (5937bytes) 2024-11-13T00:28:55,544 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42745/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731457735281/wal.1731457735337 2024-11-13T00:28:55,544 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42745/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731457735281/wal.1731457735337 after 0ms 2024-11-13T00:28:55,547 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42745/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731457735281/wal.1731457735337 took 3ms 2024-11-13T00:28:55,549 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42745/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731457735281/wal.1731457735337 so closing down 2024-11-13T00:28:55,549 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T00:28:55,550 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731457735337.temp 2024-11-13T00:28:55,552 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/recovered.edits/0000000000000000003-wal.1731457735337.temp 2024-11-13T00:28:55,552 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T00:28:55,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741898_1076 (size=5223) 2024-11-13T00:28:55,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741898_1076 (size=5223) 2024-11-13T00:28:55,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741898_1076 (size=5223) 2024-11-13T00:28:55,566 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/recovered.edits/0000000000000000003-wal.1731457735337.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T00:28:55,568 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/recovered.edits/0000000000000000003-wal.1731457735337.temp to hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/recovered.edits/0000000000000000032 2024-11-13T00:28:55,568 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 21 ms; skipped=2; WAL=hdfs://localhost:42745/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731457735281/wal.1731457735337, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-11-13T00:28:55,568 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42745/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731457735281/wal.1731457735337, journal: Splitting hdfs://localhost:42745/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731457735281/wal.1731457735337, size=5.8 K (5937bytes) at 1731457735544Finishing writing output for hdfs://localhost:42745/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731457735281/wal.1731457735337 so closing down at 1731457735549 (+5 ms)Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/recovered.edits/0000000000000000003-wal.1731457735337.temp at 1731457735552 (+3 ms)3 split writer threads finished at 1731457735552Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/recovered.edits/0000000000000000003-wal.1731457735337.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731457735566 (+14 ms)Rename recovered edits hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/recovered.edits/0000000000000000003-wal.1731457735337.temp to hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/recovered.edits/0000000000000000032 at 1731457735568 (+2 ms)Processed 32 edits across 1 Regions in 21 ms; skipped=2; WAL=hdfs://localhost:42745/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731457735281/wal.1731457735337, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1731457735568 2024-11-13T00:28:55,570 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42745/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731457735281/wal.1731457735337 to hdfs://localhost:42745/hbase/oldWALs/wal.1731457735337 2024-11-13T00:28:55,571 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/recovered.edits/0000000000000000032 2024-11-13T00:28:55,571 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T00:28:55,573 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731457735281, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:28:55,586 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731457735281/wal.1731457735573, exclude list is [], retry=0 2024-11-13T00:28:55,589 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:55,589 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:55,590 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:55,591 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731457735281/wal.1731457735573 2024-11-13T00:28:55,592 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:36255:36255)] 2024-11-13T00:28:55,592 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => aa9a2b52c7d82fdc4527eaa78c0c5ccf, NAME => 'testReplayEditsWrittenViaHRegion,,1731457735283.aa9a2b52c7d82fdc4527eaa78c0c5ccf.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:28:55,592 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731457735283.aa9a2b52c7d82fdc4527eaa78c0c5ccf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:55,592 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,592 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,594 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,595 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa9a2b52c7d82fdc4527eaa78c0c5ccf columnFamilyName a 2024-11-13T00:28:55,595 DEBUG [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:55,602 DEBUG [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/a/86f962db221c44a0b64c840903391382 2024-11-13T00:28:55,602 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] regionserver.HStore(327): Store=aa9a2b52c7d82fdc4527eaa78c0c5ccf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:55,602 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,603 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa9a2b52c7d82fdc4527eaa78c0c5ccf columnFamilyName b 2024-11-13T00:28:55,603 DEBUG [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:55,604 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] regionserver.HStore(327): Store=aa9a2b52c7d82fdc4527eaa78c0c5ccf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:55,604 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,605 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa9a2b52c7d82fdc4527eaa78c0c5ccf columnFamilyName c 2024-11-13T00:28:55,605 DEBUG [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:55,612 DEBUG [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/c/a8466e7e38e84e0fa7b47049019ca58d 2024-11-13T00:28:55,612 INFO [StoreOpener-aa9a2b52c7d82fdc4527eaa78c0c5ccf-1 {}] regionserver.HStore(327): Store=aa9a2b52c7d82fdc4527eaa78c0c5ccf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:55,612 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,613 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,615 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,615 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/recovered.edits/0000000000000000032 2024-11-13T00:28:55,619 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/recovered.edits/0000000000000000032 2024-11-13T00:28:55,619 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing aa9a2b52c7d82fdc4527eaa78c0c5ccf 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-13T00:28:55,634 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/.tmp/b/c4507e3761fd486abd34250bbc5c6981 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731457735375/Put/seqid=0 2024-11-13T00:28:55,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741900_1078 (size=5958) 2024-11-13T00:28:55,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741900_1078 (size=5958) 2024-11-13T00:28:55,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741900_1078 (size=5958) 2024-11-13T00:28:55,643 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/.tmp/b/c4507e3761fd486abd34250bbc5c6981 2024-11-13T00:28:55,648 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/.tmp/b/c4507e3761fd486abd34250bbc5c6981 as hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/b/c4507e3761fd486abd34250bbc5c6981 2024-11-13T00:28:55,654 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/b/c4507e3761fd486abd34250bbc5c6981, entries=10, sequenceid=32, filesize=5.8 K 2024-11-13T00:28:55,654 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for aa9a2b52c7d82fdc4527eaa78c0c5ccf in 35ms, sequenceid=32, compaction requested=false; wal=null 2024-11-13T00:28:55,655 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/recovered.edits/0000000000000000032 2024-11-13T00:28:55,656 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,656 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,657 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T00:28:55,658 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for aa9a2b52c7d82fdc4527eaa78c0c5ccf 2024-11-13T00:28:55,660 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/aa9a2b52c7d82fdc4527eaa78c0c5ccf/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-13T00:28:55,661 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened aa9a2b52c7d82fdc4527eaa78c0c5ccf; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63920269, jitterRate=-0.04751376807689667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T00:28:55,661 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for aa9a2b52c7d82fdc4527eaa78c0c5ccf: Writing region info on filesystem at 1731457735593Initializing all the Stores at 1731457735593Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457735594 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457735594Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457735594Obtaining lock to block concurrent updates at 1731457735619 (+25 ms)Preparing flush snapshotting stores in aa9a2b52c7d82fdc4527eaa78c0c5ccf at 1731457735619Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731457735283.aa9a2b52c7d82fdc4527eaa78c0c5ccf., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731457735620 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1731457735283.aa9a2b52c7d82fdc4527eaa78c0c5ccf. at 1731457735620Flushing aa9a2b52c7d82fdc4527eaa78c0c5ccf/b: creating writer at 1731457735620Flushing aa9a2b52c7d82fdc4527eaa78c0c5ccf/b: appending metadata at 1731457735634 (+14 ms)Flushing aa9a2b52c7d82fdc4527eaa78c0c5ccf/b: closing flushed file at 1731457735634Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f544894: reopening flushed file at 1731457735647 (+13 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for aa9a2b52c7d82fdc4527eaa78c0c5ccf in 35ms, sequenceid=32, compaction requested=false; wal=null at 1731457735654 (+7 ms)Cleaning up temporary data from old regions at 1731457735656 (+2 ms)Region opened successfully at 1731457735661 (+5 ms) 2024-11-13T00:28:55,684 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=419 (was 410) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:51976 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:52102 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:49034 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1811965340-172.17.0.2-1731457714216:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:59088 [Waiting for operation #34] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1811965340-172.17.0.2-1731457714216:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:49134 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:59234 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1811965340-172.17.0.2-1731457714216:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1127 (was 1051) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=181 (was 127) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1473 (was 1478) 2024-11-13T00:28:55,684 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-11-13T00:28:55,696 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=419, OpenFileDescriptor=1127, MaxFileDescriptor=1048576, SystemLoadAverage=181, ProcessCount=11, AvailableMemoryMB=1472 2024-11-13T00:28:55,696 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-11-13T00:28:55,711 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:55,714 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:28:55,715 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T00:28:55,718 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-04355824, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/hregion-04355824, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:28:55,731 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-04355824/hregion-04355824.1731457735718, exclude list is [], retry=0 2024-11-13T00:28:55,734 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:55,734 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:55,734 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:55,736 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-04355824/hregion-04355824.1731457735718 2024-11-13T00:28:55,737 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:41441:41441)] 2024-11-13T00:28:55,737 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5a868ad1717fbcdfc97bef8fe7e327be, NAME => 'testReplayEditsAfterAbortingFlush,,1731457735712.5a868ad1717fbcdfc97bef8fe7e327be.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42745/hbase 2024-11-13T00:28:55,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741902_1080 (size=68) 2024-11-13T00:28:55,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741902_1080 (size=68) 2024-11-13T00:28:55,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741902_1080 (size=68) 2024-11-13T00:28:55,744 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731457735712.5a868ad1717fbcdfc97bef8fe7e327be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:55,746 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,747 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a868ad1717fbcdfc97bef8fe7e327be columnFamilyName a 2024-11-13T00:28:55,747 DEBUG [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:55,747 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] regionserver.HStore(327): Store=5a868ad1717fbcdfc97bef8fe7e327be/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:55,747 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,748 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a868ad1717fbcdfc97bef8fe7e327be columnFamilyName b 2024-11-13T00:28:55,749 DEBUG [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:55,749 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] regionserver.HStore(327): Store=5a868ad1717fbcdfc97bef8fe7e327be/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:55,749 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,750 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a868ad1717fbcdfc97bef8fe7e327be columnFamilyName c 2024-11-13T00:28:55,750 DEBUG [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:55,751 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] regionserver.HStore(327): Store=5a868ad1717fbcdfc97bef8fe7e327be/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:55,751 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,752 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,752 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,753 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,753 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,753 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T00:28:55,754 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,756 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:28:55,757 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5a868ad1717fbcdfc97bef8fe7e327be; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61082412, jitterRate=-0.0898011326789856}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T00:28:55,757 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5a868ad1717fbcdfc97bef8fe7e327be: Writing region info on filesystem at 1731457735745Initializing all the Stores at 1731457735745Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457735745Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457735746 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457735746Cleaning up temporary data from old regions at 1731457735753 (+7 ms)Region opened successfully at 1731457735757 (+4 ms) 2024-11-13T00:28:55,757 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5a868ad1717fbcdfc97bef8fe7e327be, disabling compactions & flushes 2024-11-13T00:28:55,757 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731457735712.5a868ad1717fbcdfc97bef8fe7e327be. 2024-11-13T00:28:55,758 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731457735712.5a868ad1717fbcdfc97bef8fe7e327be. 2024-11-13T00:28:55,758 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731457735712.5a868ad1717fbcdfc97bef8fe7e327be. after waiting 0 ms 2024-11-13T00:28:55,758 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731457735712.5a868ad1717fbcdfc97bef8fe7e327be. 2024-11-13T00:28:55,758 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731457735712.5a868ad1717fbcdfc97bef8fe7e327be. 2024-11-13T00:28:55,758 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5a868ad1717fbcdfc97bef8fe7e327be: Waiting for close lock at 1731457735757Disabling compacts and flushes for region at 1731457735757Disabling writes for close at 1731457735758 (+1 ms)Writing region close event to WAL at 1731457735758Closed at 1731457735758 2024-11-13T00:28:55,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741901_1079 (size=93) 2024-11-13T00:28:55,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741901_1079 (size=93) 2024-11-13T00:28:55,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741901_1079 (size=93) 2024-11-13T00:28:55,762 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T00:28:55,762 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-04355824:(num 1731457735718) 2024-11-13T00:28:55,763 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T00:28:55,764 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731457735710, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:28:55,777 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731457735710/wal.1731457735764, exclude list is [], retry=0 2024-11-13T00:28:55,780 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:28:55,780 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:28:55,781 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:28:55,782 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731457735710/wal.1731457735764 2024-11-13T00:28:55,783 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:45019:45019)] 2024-11-13T00:28:55,857 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5a868ad1717fbcdfc97bef8fe7e327be, NAME => 'testReplayEditsAfterAbortingFlush,,1731457735712.5a868ad1717fbcdfc97bef8fe7e327be.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:28:55,860 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,860 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731457735712.5a868ad1717fbcdfc97bef8fe7e327be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:28:55,860 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,860 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,862 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,863 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a868ad1717fbcdfc97bef8fe7e327be columnFamilyName a 2024-11-13T00:28:55,863 DEBUG [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:55,864 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] regionserver.HStore(327): Store=5a868ad1717fbcdfc97bef8fe7e327be/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:55,864 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,865 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a868ad1717fbcdfc97bef8fe7e327be columnFamilyName b 2024-11-13T00:28:55,865 DEBUG [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:55,865 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] regionserver.HStore(327): Store=5a868ad1717fbcdfc97bef8fe7e327be/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:55,865 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,866 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a868ad1717fbcdfc97bef8fe7e327be columnFamilyName c 2024-11-13T00:28:55,866 DEBUG [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:28:55,866 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] regionserver.HStore(327): Store=5a868ad1717fbcdfc97bef8fe7e327be/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:28:55,867 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,867 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,868 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,869 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,870 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,870 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T00:28:55,872 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,873 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5a868ad1717fbcdfc97bef8fe7e327be; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67252362, jitterRate=0.002138286828994751}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T00:28:55,873 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:28:55,873 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5a868ad1717fbcdfc97bef8fe7e327be: Running coprocessor pre-open hook at 1731457735860Writing region info on filesystem at 1731457735860Initializing all the Stores at 1731457735861 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457735862 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457735862Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457735862Cleaning up temporary data from old regions at 1731457735870 (+8 ms)Running coprocessor post-open hooks at 1731457735873 (+3 ms)Region opened successfully at 1731457735873 2024-11-13T00:28:55,884 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5a868ad1717fbcdfc97bef8fe7e327be 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-13T00:28:55,885 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5a868ad1717fbcdfc97bef8fe7e327be/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:28:56,886 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5a868ad1717fbcdfc97bef8fe7e327be/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:28:57,844 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T00:28:57,887 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-13T00:28:57,887 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-13T00:28:57,887 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5a868ad1717fbcdfc97bef8fe7e327be/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:28:57,888 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-13T00:28:57,888 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-13T00:28:57,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-13T00:28:57,889 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-13T00:28:57,889 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-13T00:28:57,889 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-13T00:28:58,887 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5a868ad1717fbcdfc97bef8fe7e327be/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:28:59,888 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5a868ad1717fbcdfc97bef8fe7e327be/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:29:00,889 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5a868ad1717fbcdfc97bef8fe7e327be/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:29:01,889 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5a868ad1717fbcdfc97bef8fe7e327be/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:29:02,890 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5a868ad1717fbcdfc97bef8fe7e327be/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:29:03,891 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5a868ad1717fbcdfc97bef8fe7e327be/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:29:04,891 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5a868ad1717fbcdfc97bef8fe7e327be/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:29:04,893 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5a868ad1717fbcdfc97bef8fe7e327be: 2024-11-13T00:29:04,893 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731457735712.5a868ad1717fbcdfc97bef8fe7e327be. 2024-11-13T00:29:04,907 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5a868ad1717fbcdfc97bef8fe7e327be: 2024-11-13T00:29:04,907 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-13T00:29:04,907 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5a868ad1717fbcdfc97bef8fe7e327be, disabling compactions & flushes 2024-11-13T00:29:04,907 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731457735712.5a868ad1717fbcdfc97bef8fe7e327be. 2024-11-13T00:29:04,907 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731457735712.5a868ad1717fbcdfc97bef8fe7e327be. 2024-11-13T00:29:04,907 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731457735712.5a868ad1717fbcdfc97bef8fe7e327be. after waiting 0 ms 2024-11-13T00:29:04,907 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731457735712.5a868ad1717fbcdfc97bef8fe7e327be. 2024-11-13T00:29:04,908 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731457735712.5a868ad1717fbcdfc97bef8fe7e327be. 2024-11-13T00:29:04,908 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731457735712.5a868ad1717fbcdfc97bef8fe7e327be. 2024-11-13T00:29:04,908 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5a868ad1717fbcdfc97bef8fe7e327be: Waiting for close lock at 1731457744907Running coprocessor pre-close hooks at 1731457744907Disabling compacts and flushes for region at 1731457744907Disabling writes for close at 1731457744907Writing region close event to WAL at 1731457744908 (+1 ms)Running coprocessor post-close hooks at 1731457744908Closed at 1731457744908 2024-11-13T00:29:04,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741903_1081 (size=3941) 2024-11-13T00:29:04,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741903_1081 (size=3941) 2024-11-13T00:29:04,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741903_1081 (size=3941) 2024-11-13T00:29:04,927 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42745/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731457735710/wal.1731457735764, size=3.8 K (3941bytes) 2024-11-13T00:29:04,927 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42745/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731457735710/wal.1731457735764 2024-11-13T00:29:04,927 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42745/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731457735710/wal.1731457735764 after 0ms 2024-11-13T00:29:04,930 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42745/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731457735710/wal.1731457735764 took 3ms 2024-11-13T00:29:04,932 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42745/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731457735710/wal.1731457735764 so closing down 2024-11-13T00:29:04,932 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T00:29:04,933 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731457735764.temp 2024-11-13T00:29:04,934 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/recovered.edits/0000000000000000004-wal.1731457735764.temp 2024-11-13T00:29:04,934 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T00:29:04,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741904_1082 (size=2983) 2024-11-13T00:29:04,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741904_1082 (size=2983) 2024-11-13T00:29:04,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741904_1082 (size=2983) 2024-11-13T00:29:04,946 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/recovered.edits/0000000000000000004-wal.1731457735764.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-13T00:29:04,948 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/recovered.edits/0000000000000000004-wal.1731457735764.temp to hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/recovered.edits/0000000000000000026 2024-11-13T00:29:04,948 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 18 ms; skipped=3; WAL=hdfs://localhost:42745/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731457735710/wal.1731457735764, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-11-13T00:29:04,948 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42745/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731457735710/wal.1731457735764, journal: Splitting hdfs://localhost:42745/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731457735710/wal.1731457735764, size=3.8 K (3941bytes) at 1731457744927Finishing writing output for hdfs://localhost:42745/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731457735710/wal.1731457735764 so closing down at 1731457744932 (+5 ms)Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/recovered.edits/0000000000000000004-wal.1731457735764.temp at 1731457744934 (+2 ms)3 split writer threads finished at 1731457744934Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/recovered.edits/0000000000000000004-wal.1731457735764.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731457744946 (+12 ms)Rename recovered edits hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/recovered.edits/0000000000000000004-wal.1731457735764.temp to hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/recovered.edits/0000000000000000026 at 1731457744948 (+2 ms)Processed 23 edits across 1 Regions in 18 ms; skipped=3; WAL=hdfs://localhost:42745/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731457735710/wal.1731457735764, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1731457744948 2024-11-13T00:29:04,950 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42745/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731457735710/wal.1731457735764 to hdfs://localhost:42745/hbase/oldWALs/wal.1731457735764 2024-11-13T00:29:04,950 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/recovered.edits/0000000000000000026 2024-11-13T00:29:04,951 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T00:29:04,952 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731457735710, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:29:04,965 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731457735710/wal.1731457744952, exclude list is [], retry=0 2024-11-13T00:29:04,968 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:29:04,968 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:29:04,968 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:29:04,970 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731457735710/wal.1731457744952 2024-11-13T00:29:04,970 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:36255:36255)] 2024-11-13T00:29:04,971 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5a868ad1717fbcdfc97bef8fe7e327be, NAME => 'testReplayEditsAfterAbortingFlush,,1731457735712.5a868ad1717fbcdfc97bef8fe7e327be.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:29:04,971 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:29:04,971 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731457735712.5a868ad1717fbcdfc97bef8fe7e327be.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:29:04,971 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:29:04,971 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:29:04,973 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:29:04,974 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a868ad1717fbcdfc97bef8fe7e327be columnFamilyName a 2024-11-13T00:29:04,974 DEBUG [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:04,974 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] regionserver.HStore(327): Store=5a868ad1717fbcdfc97bef8fe7e327be/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:04,974 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:29:04,975 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a868ad1717fbcdfc97bef8fe7e327be columnFamilyName b 2024-11-13T00:29:04,975 DEBUG [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:04,976 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] regionserver.HStore(327): Store=5a868ad1717fbcdfc97bef8fe7e327be/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:04,976 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:29:04,976 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a868ad1717fbcdfc97bef8fe7e327be columnFamilyName c 2024-11-13T00:29:04,976 DEBUG [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:04,977 INFO [StoreOpener-5a868ad1717fbcdfc97bef8fe7e327be-1 {}] regionserver.HStore(327): Store=5a868ad1717fbcdfc97bef8fe7e327be/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:04,977 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:29:04,978 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:29:04,979 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:29:04,979 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/recovered.edits/0000000000000000026 2024-11-13T00:29:04,982 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/recovered.edits/0000000000000000026 2024-11-13T00:29:04,983 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5a868ad1717fbcdfc97bef8fe7e327be 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-13T00:29:04,997 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/.tmp/a/bf08aeae4f634a988aab25b7cb2f778c is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731457744899/Put/seqid=0 2024-11-13T00:29:05,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741906_1084 (size=5523) 2024-11-13T00:29:05,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741906_1084 (size=5523) 2024-11-13T00:29:05,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741906_1084 (size=5523) 2024-11-13T00:29:05,007 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/.tmp/a/bf08aeae4f634a988aab25b7cb2f778c 2024-11-13T00:29:05,027 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/.tmp/b/612d2f42a35649cbb2cd24c5125295e3 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731457744893/Put/seqid=0 2024-11-13T00:29:05,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741907_1085 (size=5524) 2024-11-13T00:29:05,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741907_1085 (size=5524) 2024-11-13T00:29:05,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741907_1085 (size=5524) 2024-11-13T00:29:05,037 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/.tmp/b/612d2f42a35649cbb2cd24c5125295e3 2024-11-13T00:29:05,062 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/.tmp/c/95b9a8bcf40d484ebdd02c742ccbebe0 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731457744896/Put/seqid=0 2024-11-13T00:29:05,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741908_1086 (size=5457) 2024-11-13T00:29:05,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741908_1086 (size=5457) 2024-11-13T00:29:05,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741908_1086 (size=5457) 2024-11-13T00:29:05,074 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/.tmp/c/95b9a8bcf40d484ebdd02c742ccbebe0 2024-11-13T00:29:05,082 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/.tmp/a/bf08aeae4f634a988aab25b7cb2f778c as hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/a/bf08aeae4f634a988aab25b7cb2f778c 2024-11-13T00:29:05,088 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/a/bf08aeae4f634a988aab25b7cb2f778c, entries=7, sequenceid=26, filesize=5.4 K 2024-11-13T00:29:05,089 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/.tmp/b/612d2f42a35649cbb2cd24c5125295e3 as hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/b/612d2f42a35649cbb2cd24c5125295e3 2024-11-13T00:29:05,095 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/b/612d2f42a35649cbb2cd24c5125295e3, entries=7, sequenceid=26, filesize=5.4 K 2024-11-13T00:29:05,097 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/.tmp/c/95b9a8bcf40d484ebdd02c742ccbebe0 as hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/c/95b9a8bcf40d484ebdd02c742ccbebe0 2024-11-13T00:29:05,104 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/c/95b9a8bcf40d484ebdd02c742ccbebe0, entries=6, sequenceid=26, filesize=5.3 K 2024-11-13T00:29:05,104 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 5a868ad1717fbcdfc97bef8fe7e327be in 122ms, sequenceid=26, compaction requested=false; wal=null 2024-11-13T00:29:05,105 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/recovered.edits/0000000000000000026 2024-11-13T00:29:05,106 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:29:05,106 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:29:05,107 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T00:29:05,109 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:29:05,112 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testReplayEditsAfterAbortingFlush/5a868ad1717fbcdfc97bef8fe7e327be/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-13T00:29:05,113 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5a868ad1717fbcdfc97bef8fe7e327be; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62863857, jitterRate=-0.06325553357601166}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T00:29:05,113 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5a868ad1717fbcdfc97bef8fe7e327be 2024-11-13T00:29:05,114 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5a868ad1717fbcdfc97bef8fe7e327be: Running coprocessor pre-open hook at 1731457744972Writing region info on filesystem at 1731457744972Initializing all the Stores at 1731457744972Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457744972Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457744972Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457744972Obtaining lock to block concurrent updates at 1731457744983 (+11 ms)Preparing flush snapshotting stores in 5a868ad1717fbcdfc97bef8fe7e327be at 1731457744983Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731457735712.5a868ad1717fbcdfc97bef8fe7e327be., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731457744983Flushing stores of testReplayEditsAfterAbortingFlush,,1731457735712.5a868ad1717fbcdfc97bef8fe7e327be. at 1731457744983Flushing 5a868ad1717fbcdfc97bef8fe7e327be/a: creating writer at 1731457744983Flushing 5a868ad1717fbcdfc97bef8fe7e327be/a: appending metadata at 1731457744997 (+14 ms)Flushing 5a868ad1717fbcdfc97bef8fe7e327be/a: closing flushed file at 1731457744997Flushing 5a868ad1717fbcdfc97bef8fe7e327be/b: creating writer at 1731457745013 (+16 ms)Flushing 5a868ad1717fbcdfc97bef8fe7e327be/b: appending metadata at 1731457745027 (+14 ms)Flushing 5a868ad1717fbcdfc97bef8fe7e327be/b: closing flushed file at 1731457745027Flushing 5a868ad1717fbcdfc97bef8fe7e327be/c: creating writer at 1731457745044 (+17 ms)Flushing 5a868ad1717fbcdfc97bef8fe7e327be/c: appending metadata at 1731457745061 (+17 ms)Flushing 5a868ad1717fbcdfc97bef8fe7e327be/c: closing flushed file at 1731457745061Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a8512f7: reopening flushed file at 1731457745081 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@680732ef: reopening flushed file at 1731457745088 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c6a4e50: reopening flushed file at 1731457745095 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 5a868ad1717fbcdfc97bef8fe7e327be in 122ms, sequenceid=26, compaction requested=false; wal=null at 1731457745104 (+9 ms)Cleaning up temporary data from old regions at 1731457745106 (+2 ms)Running coprocessor post-open hooks at 1731457745113 (+7 ms)Region opened successfully at 1731457745114 (+1 ms) 2024-11-13T00:29:05,143 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415 (was 419), OpenFileDescriptor=1181 (was 1127) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=183 (was 181) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1392 (was 1472) 2024-11-13T00:29:05,143 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1181 is superior to 1024 2024-11-13T00:29:05,158 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=415, OpenFileDescriptor=1181, MaxFileDescriptor=1048576, SystemLoadAverage=183, ProcessCount=11, AvailableMemoryMB=1384 2024-11-13T00:29:05,158 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1181 is superior to 1024 2024-11-13T00:29:05,174 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:29:05,176 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:29:05,177 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T00:29:05,180 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-11538831, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/hregion-11538831, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:29:05,193 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-11538831/hregion-11538831.1731457745180, exclude list is [], retry=0 2024-11-13T00:29:05,195 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:29:05,195 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:29:05,196 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:29:05,198 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-11538831/hregion-11538831.1731457745180 2024-11-13T00:29:05,198 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:41441:41441)] 2024-11-13T00:29:05,198 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1e80ba5114c5be2977d8cf90d7adcc2a, NAME => 'testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42745/hbase 2024-11-13T00:29:05,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741910_1088 (size=61) 2024-11-13T00:29:05,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741910_1088 (size=61) 2024-11-13T00:29:05,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741910_1088 (size=61) 2024-11-13T00:29:05,207 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:29:05,212 INFO [StoreOpener-1e80ba5114c5be2977d8cf90d7adcc2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,214 INFO [StoreOpener-1e80ba5114c5be2977d8cf90d7adcc2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e80ba5114c5be2977d8cf90d7adcc2a columnFamilyName a 2024-11-13T00:29:05,214 DEBUG [StoreOpener-1e80ba5114c5be2977d8cf90d7adcc2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:05,214 INFO [StoreOpener-1e80ba5114c5be2977d8cf90d7adcc2a-1 {}] regionserver.HStore(327): Store=1e80ba5114c5be2977d8cf90d7adcc2a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:05,215 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,215 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,216 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,216 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,217 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,218 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,221 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:29:05,222 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1e80ba5114c5be2977d8cf90d7adcc2a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60428512, jitterRate=-0.09954500198364258}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T00:29:05,223 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1e80ba5114c5be2977d8cf90d7adcc2a: Writing region info on filesystem at 1731457745208Initializing all the Stores at 1731457745208Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457745208Cleaning up temporary data from old regions at 1731457745217 (+9 ms)Region opened successfully at 1731457745223 (+6 ms) 2024-11-13T00:29:05,223 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1e80ba5114c5be2977d8cf90d7adcc2a, disabling compactions & flushes 2024-11-13T00:29:05,223 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a. 2024-11-13T00:29:05,223 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a. 2024-11-13T00:29:05,223 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a. after waiting 0 ms 2024-11-13T00:29:05,223 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a. 2024-11-13T00:29:05,224 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a. 2024-11-13T00:29:05,224 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1e80ba5114c5be2977d8cf90d7adcc2a: Waiting for close lock at 1731457745223Disabling compacts and flushes for region at 1731457745223Disabling writes for close at 1731457745223Writing region close event to WAL at 1731457745224 (+1 ms)Closed at 1731457745224 2024-11-13T00:29:05,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741909_1087 (size=93) 2024-11-13T00:29:05,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741909_1087 (size=93) 2024-11-13T00:29:05,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741909_1087 (size=93) 2024-11-13T00:29:05,235 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T00:29:05,236 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-11538831:(num 1731457745180) 2024-11-13T00:29:05,236 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T00:29:05,238 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/testdatalosswheninputerror-manual,16010,1731457745174, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:29:05,252 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731457745174/wal.1731457745238, exclude list is [], retry=0 2024-11-13T00:29:05,255 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:29:05,255 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:29:05,256 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:29:05,258 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731457745174/wal.1731457745238 2024-11-13T00:29:05,258 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:45019:45019)] 2024-11-13T00:29:05,258 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1e80ba5114c5be2977d8cf90d7adcc2a, NAME => 'testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:29:05,259 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:29:05,259 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,259 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,260 INFO [StoreOpener-1e80ba5114c5be2977d8cf90d7adcc2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,261 INFO [StoreOpener-1e80ba5114c5be2977d8cf90d7adcc2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e80ba5114c5be2977d8cf90d7adcc2a columnFamilyName a 2024-11-13T00:29:05,261 DEBUG [StoreOpener-1e80ba5114c5be2977d8cf90d7adcc2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:05,262 INFO [StoreOpener-1e80ba5114c5be2977d8cf90d7adcc2a-1 {}] regionserver.HStore(327): Store=1e80ba5114c5be2977d8cf90d7adcc2a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:05,262 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,263 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,264 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,265 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,265 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,266 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,267 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1e80ba5114c5be2977d8cf90d7adcc2a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70357124, jitterRate=0.04840284585952759}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T00:29:05,268 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1e80ba5114c5be2977d8cf90d7adcc2a: Writing region info on filesystem at 1731457745259Initializing all the Stores at 1731457745260 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457745260Cleaning up temporary data from old regions at 1731457745265 (+5 ms)Region opened successfully at 1731457745268 (+3 ms) 2024-11-13T00:29:05,281 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1e80ba5114c5be2977d8cf90d7adcc2a, disabling compactions & flushes 2024-11-13T00:29:05,281 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a. 2024-11-13T00:29:05,281 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a. 2024-11-13T00:29:05,281 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a. after waiting 0 ms 2024-11-13T00:29:05,281 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a. 2024-11-13T00:29:05,281 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a. 2024-11-13T00:29:05,281 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a. 2024-11-13T00:29:05,281 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1e80ba5114c5be2977d8cf90d7adcc2a: Waiting for close lock at 1731457745281Disabling compacts and flushes for region at 1731457745281Disabling writes for close at 1731457745281Writing region close event to WAL at 1731457745281Closed at 1731457745281 2024-11-13T00:29:05,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741911_1089 (size=1623) 2024-11-13T00:29:05,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741911_1089 (size=1623) 2024-11-13T00:29:05,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741911_1089 (size=1623) 2024-11-13T00:29:05,307 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42745/hbase/WALs/testdatalosswheninputerror-manual,16010,1731457745174/wal.1731457745238, size=1.6 K (1623bytes) 2024-11-13T00:29:05,307 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42745/hbase/WALs/testdatalosswheninputerror-manual,16010,1731457745174/wal.1731457745238 2024-11-13T00:29:05,307 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42745/hbase/WALs/testdatalosswheninputerror-manual,16010,1731457745174/wal.1731457745238 after 0ms 2024-11-13T00:29:05,310 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42745/hbase/WALs/testdatalosswheninputerror-manual,16010,1731457745174/wal.1731457745238 took 3ms 2024-11-13T00:29:05,312 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42745/hbase/WALs/testdatalosswheninputerror-manual,16010,1731457745174/wal.1731457745238 so closing down 2024-11-13T00:29:05,312 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T00:29:05,313 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731457745238.temp 2024-11-13T00:29:05,314 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/recovered.edits/0000000000000000003-wal.1731457745238.temp 2024-11-13T00:29:05,315 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T00:29:05,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741912_1090 (size=1623) 2024-11-13T00:29:05,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741912_1090 (size=1623) 2024-11-13T00:29:05,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741912_1090 (size=1623) 2024-11-13T00:29:05,330 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/recovered.edits/0000000000000000003-wal.1731457745238.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-13T00:29:05,331 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/recovered.edits/0000000000000000003-wal.1731457745238.temp to hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/recovered.edits/0000000000000000012 2024-11-13T00:29:05,331 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:42745/hbase/WALs/testdatalosswheninputerror-manual,16010,1731457745174/wal.1731457745238, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-11-13T00:29:05,332 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42745/hbase/WALs/testdatalosswheninputerror-manual,16010,1731457745174/wal.1731457745238, journal: Splitting hdfs://localhost:42745/hbase/WALs/testdatalosswheninputerror-manual,16010,1731457745174/wal.1731457745238, size=1.6 K (1623bytes) at 1731457745307Finishing writing output for hdfs://localhost:42745/hbase/WALs/testdatalosswheninputerror-manual,16010,1731457745174/wal.1731457745238 so closing down at 1731457745312 (+5 ms)Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/recovered.edits/0000000000000000003-wal.1731457745238.temp at 1731457745314 (+2 ms)3 split writer threads finished at 1731457745315 (+1 ms)Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/recovered.edits/0000000000000000003-wal.1731457745238.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731457745330 (+15 ms)Rename recovered edits hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/recovered.edits/0000000000000000003-wal.1731457745238.temp to hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/recovered.edits/0000000000000000012 at 1731457745331 (+1 ms)Processed 10 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:42745/hbase/WALs/testdatalosswheninputerror-manual,16010,1731457745174/wal.1731457745238, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1731457745332 (+1 ms) 2024-11-13T00:29:05,334 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42745/hbase/WALs/testdatalosswheninputerror-manual,16010,1731457745174/wal.1731457745238 to hdfs://localhost:42745/hbase/oldWALs/wal.1731457745238 2024-11-13T00:29:05,335 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/recovered.edits/0000000000000000012 2024-11-13T00:29:05,709 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T00:29:05,711 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/testdatalosswheninputerror-manual,16010,1731457745174, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:29:05,725 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731457745174/wal.1731457745712, exclude list is [], retry=0 2024-11-13T00:29:05,727 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:29:05,728 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:29:05,728 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:29:05,730 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731457745174/wal.1731457745712 2024-11-13T00:29:05,731 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:36255:36255)] 2024-11-13T00:29:05,731 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1e80ba5114c5be2977d8cf90d7adcc2a, NAME => 'testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:29:05,732 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:29:05,732 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,732 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,736 INFO [StoreOpener-1e80ba5114c5be2977d8cf90d7adcc2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,738 INFO [StoreOpener-1e80ba5114c5be2977d8cf90d7adcc2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e80ba5114c5be2977d8cf90d7adcc2a columnFamilyName a 2024-11-13T00:29:05,739 DEBUG [StoreOpener-1e80ba5114c5be2977d8cf90d7adcc2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:05,740 INFO [StoreOpener-1e80ba5114c5be2977d8cf90d7adcc2a-1 {}] regionserver.HStore(327): Store=1e80ba5114c5be2977d8cf90d7adcc2a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:05,740 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,741 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,742 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,743 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/recovered.edits/0000000000000000012 2024-11-13T00:29:05,746 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/recovered.edits/0000000000000000012 2024-11-13T00:29:05,746 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1e80ba5114c5be2977d8cf90d7adcc2a 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-13T00:29:05,775 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/.tmp/a/7f5a93dc9ed047b6bdcc156ff0a8c58c is 79, key is testDatalossWhenInputError/a:x0/1731457745268/Put/seqid=0 2024-11-13T00:29:05,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741914_1092 (size=5808) 2024-11-13T00:29:05,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741914_1092 (size=5808) 2024-11-13T00:29:05,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741914_1092 (size=5808) 2024-11-13T00:29:05,789 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/.tmp/a/7f5a93dc9ed047b6bdcc156ff0a8c58c 2024-11-13T00:29:05,799 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/.tmp/a/7f5a93dc9ed047b6bdcc156ff0a8c58c as hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/a/7f5a93dc9ed047b6bdcc156ff0a8c58c 2024-11-13T00:29:05,809 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/a/7f5a93dc9ed047b6bdcc156ff0a8c58c, entries=10, sequenceid=12, filesize=5.7 K 2024-11-13T00:29:05,809 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 1e80ba5114c5be2977d8cf90d7adcc2a in 63ms, sequenceid=12, compaction requested=false; wal=null 2024-11-13T00:29:05,810 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/recovered.edits/0000000000000000012 2024-11-13T00:29:05,811 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,811 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,815 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,817 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-13T00:29:05,818 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1e80ba5114c5be2977d8cf90d7adcc2a; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66192468, jitterRate=-0.013655364513397217}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T00:29:05,819 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1e80ba5114c5be2977d8cf90d7adcc2a: Writing region info on filesystem at 1731457745732Initializing all the Stores at 1731457745734 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457745734Obtaining lock to block concurrent updates at 1731457745746 (+12 ms)Preparing flush snapshotting stores in 1e80ba5114c5be2977d8cf90d7adcc2a at 1731457745746Finished memstore snapshotting testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731457745746Flushing stores of testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a. at 1731457745746Flushing 1e80ba5114c5be2977d8cf90d7adcc2a/a: creating writer at 1731457745746Flushing 1e80ba5114c5be2977d8cf90d7adcc2a/a: appending metadata at 1731457745774 (+28 ms)Flushing 1e80ba5114c5be2977d8cf90d7adcc2a/a: closing flushed file at 1731457745774Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74f720fa: reopening flushed file at 1731457745798 (+24 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 1e80ba5114c5be2977d8cf90d7adcc2a in 63ms, sequenceid=12, compaction requested=false; wal=null at 1731457745809 (+11 ms)Cleaning up temporary data from old regions at 1731457745811 (+2 ms)Region opened successfully at 1731457745819 (+8 ms) 2024-11-13T00:29:05,823 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1e80ba5114c5be2977d8cf90d7adcc2a, NAME => 'testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:29:05,823 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731457745175.1e80ba5114c5be2977d8cf90d7adcc2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:29:05,824 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,824 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,825 INFO [StoreOpener-1e80ba5114c5be2977d8cf90d7adcc2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,826 INFO [StoreOpener-1e80ba5114c5be2977d8cf90d7adcc2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e80ba5114c5be2977d8cf90d7adcc2a columnFamilyName a 2024-11-13T00:29:05,826 DEBUG [StoreOpener-1e80ba5114c5be2977d8cf90d7adcc2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:05,832 DEBUG [StoreOpener-1e80ba5114c5be2977d8cf90d7adcc2a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/a/7f5a93dc9ed047b6bdcc156ff0a8c58c 2024-11-13T00:29:05,832 INFO [StoreOpener-1e80ba5114c5be2977d8cf90d7adcc2a-1 {}] regionserver.HStore(327): Store=1e80ba5114c5be2977d8cf90d7adcc2a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:05,832 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,833 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,834 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,835 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,835 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,836 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1e80ba5114c5be2977d8cf90d7adcc2a 2024-11-13T00:29:05,839 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testDatalossWhenInputError/1e80ba5114c5be2977d8cf90d7adcc2a/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-13T00:29:05,840 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1e80ba5114c5be2977d8cf90d7adcc2a; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63044214, jitterRate=-0.060568004846572876}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T00:29:05,840 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1e80ba5114c5be2977d8cf90d7adcc2a: Writing region info on filesystem at 1731457745824Initializing all the Stores at 1731457745825 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457745825Cleaning up temporary data from old regions at 1731457745835 (+10 ms)Region opened successfully at 1731457745840 (+5 ms) 2024-11-13T00:29:05,861 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=425 (was 415) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:44182 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:43328 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1811965340-172.17.0.2-1731457714216:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:43282 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:55690 [Waiting for operation #29] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:44244 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:55808 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1811965340-172.17.0.2-1731457714216:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1811965340-172.17.0.2-1731457714216:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1263 (was 1181) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=208 (was 183) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1384 (was 1384) 2024-11-13T00:29:05,861 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1263 is superior to 1024 2024-11-13T00:29:05,876 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=425, OpenFileDescriptor=1263, MaxFileDescriptor=1048576, SystemLoadAverage=208, ProcessCount=11, AvailableMemoryMB=1383 2024-11-13T00:29:05,876 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1263 is superior to 1024 2024-11-13T00:29:05,898 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:29:05,900 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:29:05,900 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T00:29:05,903 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-67701633, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/hregion-67701633, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:29:05,917 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-67701633/hregion-67701633.1731457745904, exclude list is [], retry=0 2024-11-13T00:29:05,920 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:29:05,920 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:29:05,921 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:29:05,925 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-67701633/hregion-67701633.1731457745904 2024-11-13T00:29:05,928 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:45019:45019)] 2024-11-13T00:29:05,929 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ba4f5827573463818d730cc47ef7a77e, NAME => 'testCompactedBulkLoadedFiles,,1731457745899.ba4f5827573463818d730cc47ef7a77e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42745/hbase 2024-11-13T00:29:05,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741916_1094 (size=63) 2024-11-13T00:29:05,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741916_1094 (size=63) 2024-11-13T00:29:05,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741916_1094 (size=63) 2024-11-13T00:29:05,947 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731457745899.ba4f5827573463818d730cc47ef7a77e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:29:05,949 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:05,950 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba4f5827573463818d730cc47ef7a77e columnFamilyName a 2024-11-13T00:29:05,951 DEBUG [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:05,951 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.HStore(327): Store=ba4f5827573463818d730cc47ef7a77e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:05,951 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:05,953 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba4f5827573463818d730cc47ef7a77e columnFamilyName b 2024-11-13T00:29:05,953 DEBUG [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:05,953 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.HStore(327): Store=ba4f5827573463818d730cc47ef7a77e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:05,954 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:05,955 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba4f5827573463818d730cc47ef7a77e columnFamilyName c 2024-11-13T00:29:05,955 DEBUG [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:05,955 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.HStore(327): Store=ba4f5827573463818d730cc47ef7a77e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:05,956 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:05,956 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:05,957 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:05,958 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:05,958 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:05,959 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T00:29:05,960 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:05,962 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:29:05,962 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ba4f5827573463818d730cc47ef7a77e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68424080, jitterRate=0.01959824562072754}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T00:29:05,963 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ba4f5827573463818d730cc47ef7a77e: Writing region info on filesystem at 1731457745948Initializing all the Stores at 1731457745949 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457745949Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457745949Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457745949Cleaning up temporary data from old regions at 1731457745958 (+9 ms)Region opened successfully at 1731457745963 (+5 ms) 2024-11-13T00:29:05,963 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ba4f5827573463818d730cc47ef7a77e, disabling compactions & flushes 2024-11-13T00:29:05,963 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731457745899.ba4f5827573463818d730cc47ef7a77e. 2024-11-13T00:29:05,963 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731457745899.ba4f5827573463818d730cc47ef7a77e. 2024-11-13T00:29:05,963 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731457745899.ba4f5827573463818d730cc47ef7a77e. after waiting 0 ms 2024-11-13T00:29:05,963 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731457745899.ba4f5827573463818d730cc47ef7a77e. 2024-11-13T00:29:05,963 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731457745899.ba4f5827573463818d730cc47ef7a77e. 2024-11-13T00:29:05,963 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ba4f5827573463818d730cc47ef7a77e: Waiting for close lock at 1731457745963Disabling compacts and flushes for region at 1731457745963Disabling writes for close at 1731457745963Writing region close event to WAL at 1731457745963Closed at 1731457745963 2024-11-13T00:29:05,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741915_1093 (size=93) 2024-11-13T00:29:05,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741915_1093 (size=93) 2024-11-13T00:29:05,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741915_1093 (size=93) 2024-11-13T00:29:05,968 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T00:29:05,968 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-67701633:(num 1731457745904) 2024-11-13T00:29:05,968 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T00:29:05,970 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:29:05,983 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897/wal.1731457745970, exclude list is [], retry=0 2024-11-13T00:29:05,985 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:29:05,986 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:29:05,986 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:29:05,989 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897/wal.1731457745970 2024-11-13T00:29:05,990 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:45019:45019)] 2024-11-13T00:29:05,991 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ba4f5827573463818d730cc47ef7a77e, NAME => 'testCompactedBulkLoadedFiles,,1731457745899.ba4f5827573463818d730cc47ef7a77e.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:29:05,991 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731457745899.ba4f5827573463818d730cc47ef7a77e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:29:05,991 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:05,991 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:05,992 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:05,993 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba4f5827573463818d730cc47ef7a77e columnFamilyName a 2024-11-13T00:29:05,993 DEBUG [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:05,994 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.HStore(327): Store=ba4f5827573463818d730cc47ef7a77e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:05,994 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:05,995 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba4f5827573463818d730cc47ef7a77e columnFamilyName b 2024-11-13T00:29:05,995 DEBUG [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:05,995 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.HStore(327): Store=ba4f5827573463818d730cc47ef7a77e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:05,996 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:05,997 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba4f5827573463818d730cc47ef7a77e columnFamilyName c 2024-11-13T00:29:05,997 DEBUG [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:05,997 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.HStore(327): Store=ba4f5827573463818d730cc47ef7a77e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:05,997 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:05,998 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:05,999 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:06,000 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:06,000 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:06,001 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T00:29:06,003 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:06,004 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ba4f5827573463818d730cc47ef7a77e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68032378, jitterRate=0.013761430978775024}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T00:29:06,004 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ba4f5827573463818d730cc47ef7a77e: Writing region info on filesystem at 1731457745991Initializing all the Stores at 1731457745992 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457745992Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457745992Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457745992Cleaning up temporary data from old regions at 1731457746000 (+8 ms)Region opened successfully at 1731457746004 (+4 ms) 2024-11-13T00:29:06,009 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731457746008/Put/seqid=0 2024-11-13T00:29:06,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741918_1096 (size=4875) 2024-11-13T00:29:06,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741918_1096 (size=4875) 2024-11-13T00:29:06,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741918_1096 (size=4875) 2024-11-13T00:29:06,032 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731457746032/Put/seqid=0 2024-11-13T00:29:06,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741919_1097 (size=4875) 2024-11-13T00:29:06,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741919_1097 (size=4875) 2024-11-13T00:29:06,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741919_1097 (size=4875) 2024-11-13T00:29:06,048 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731457746048/Put/seqid=0 2024-11-13T00:29:06,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741920_1098 (size=4875) 2024-11-13T00:29:06,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741920_1098 (size=4875) 2024-11-13T00:29:06,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741920_1098 (size=4875) 2024-11-13T00:29:06,065 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:42745/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in ba4f5827573463818d730cc47ef7a77e/a 2024-11-13T00:29:06,069 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-13T00:29:06,069 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T00:29:06,069 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:42745/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in ba4f5827573463818d730cc47ef7a77e/a 2024-11-13T00:29:06,073 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-13T00:29:06,073 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T00:29:06,073 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:42745/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in ba4f5827573463818d730cc47ef7a77e/a 2024-11-13T00:29:06,076 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-13T00:29:06,076 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T00:29:06,076 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ba4f5827573463818d730cc47ef7a77e 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-13T00:29:06,098 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/.tmp/a/cd541bdc67fe47428ca1bcf3290b0ec2 is 55, key is testCompactedBulkLoadedFiles/a:a/1731457746005/Put/seqid=0 2024-11-13T00:29:06,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741921_1099 (size=5107) 2024-11-13T00:29:06,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741921_1099 (size=5107) 2024-11-13T00:29:06,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741921_1099 (size=5107) 2024-11-13T00:29:06,111 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/.tmp/a/cd541bdc67fe47428ca1bcf3290b0ec2 2024-11-13T00:29:06,118 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/.tmp/a/cd541bdc67fe47428ca1bcf3290b0ec2 as hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/cd541bdc67fe47428ca1bcf3290b0ec2 2024-11-13T00:29:06,124 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/cd541bdc67fe47428ca1bcf3290b0ec2, entries=1, sequenceid=4, filesize=5.0 K 2024-11-13T00:29:06,126 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for ba4f5827573463818d730cc47ef7a77e in 50ms, sequenceid=4, compaction requested=false 2024-11-13T00:29:06,126 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ba4f5827573463818d730cc47ef7a77e: 2024-11-13T00:29:06,127 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/be6ebe73891e4f24acf7249beabea75a_SeqId_4_ 2024-11-13T00:29:06,129 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/859be860796a44098f51cdc0d8d25b1a_SeqId_4_ 2024-11-13T00:29:06,130 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/efed154da6d94c2589e2b6e4d45dbfc1_SeqId_4_ 2024-11-13T00:29:06,131 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:42745/hbase/testCompactedBulkLoadedFiles/hfile0 into ba4f5827573463818d730cc47ef7a77e/a as hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/be6ebe73891e4f24acf7249beabea75a_SeqId_4_ - updating store file list. 2024-11-13T00:29:06,144 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for be6ebe73891e4f24acf7249beabea75a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T00:29:06,144 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/be6ebe73891e4f24acf7249beabea75a_SeqId_4_ into ba4f5827573463818d730cc47ef7a77e/a 2024-11-13T00:29:06,144 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:42745/hbase/testCompactedBulkLoadedFiles/hfile0 into ba4f5827573463818d730cc47ef7a77e/a (new location: hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/be6ebe73891e4f24acf7249beabea75a_SeqId_4_) 2024-11-13T00:29:06,147 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:42745/hbase/testCompactedBulkLoadedFiles/hfile1 into ba4f5827573463818d730cc47ef7a77e/a as hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/859be860796a44098f51cdc0d8d25b1a_SeqId_4_ - updating store file list. 2024-11-13T00:29:06,153 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 859be860796a44098f51cdc0d8d25b1a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T00:29:06,153 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/859be860796a44098f51cdc0d8d25b1a_SeqId_4_ into ba4f5827573463818d730cc47ef7a77e/a 2024-11-13T00:29:06,153 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:42745/hbase/testCompactedBulkLoadedFiles/hfile1 into ba4f5827573463818d730cc47ef7a77e/a (new location: hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/859be860796a44098f51cdc0d8d25b1a_SeqId_4_) 2024-11-13T00:29:06,154 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:42745/hbase/testCompactedBulkLoadedFiles/hfile2 into ba4f5827573463818d730cc47ef7a77e/a as hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/efed154da6d94c2589e2b6e4d45dbfc1_SeqId_4_ - updating store file list. 2024-11-13T00:29:06,160 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for efed154da6d94c2589e2b6e4d45dbfc1_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T00:29:06,160 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/efed154da6d94c2589e2b6e4d45dbfc1_SeqId_4_ into ba4f5827573463818d730cc47ef7a77e/a 2024-11-13T00:29:06,160 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:42745/hbase/testCompactedBulkLoadedFiles/hfile2 into ba4f5827573463818d730cc47ef7a77e/a (new location: hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/efed154da6d94c2589e2b6e4d45dbfc1_SeqId_4_) 2024-11-13T00:29:06,168 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T00:29:06,169 DEBUG [Time-limited test {}] regionserver.HStore(1541): ba4f5827573463818d730cc47ef7a77e/a is initiating major compaction (all files) 2024-11-13T00:29:06,169 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of ba4f5827573463818d730cc47ef7a77e/a in testCompactedBulkLoadedFiles,,1731457745899.ba4f5827573463818d730cc47ef7a77e. 2024-11-13T00:29:06,169 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/cd541bdc67fe47428ca1bcf3290b0ec2, hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/be6ebe73891e4f24acf7249beabea75a_SeqId_4_, hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/859be860796a44098f51cdc0d8d25b1a_SeqId_4_, hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/efed154da6d94c2589e2b6e4d45dbfc1_SeqId_4_] into tmpdir=hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/.tmp, totalSize=19.3 K 2024-11-13T00:29:06,169 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting cd541bdc67fe47428ca1bcf3290b0ec2, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731457746005 2024-11-13T00:29:06,170 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting be6ebe73891e4f24acf7249beabea75a_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T00:29:06,170 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 859be860796a44098f51cdc0d8d25b1a_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T00:29:06,171 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting efed154da6d94c2589e2b6e4d45dbfc1_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T00:29:06,184 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/.tmp/a/de153366d6fe450eb7f24f087110f936 is 55, key is testCompactedBulkLoadedFiles/a:a/1731457746005/Put/seqid=0 2024-11-13T00:29:06,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741922_1100 (size=6154) 2024-11-13T00:29:06,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741922_1100 (size=6154) 2024-11-13T00:29:06,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741922_1100 (size=6154) 2024-11-13T00:29:06,325 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T00:29:06,600 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/.tmp/a/de153366d6fe450eb7f24f087110f936 as hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/de153366d6fe450eb7f24f087110f936 2024-11-13T00:29:06,606 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in ba4f5827573463818d730cc47ef7a77e/a of ba4f5827573463818d730cc47ef7a77e into de153366d6fe450eb7f24f087110f936(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T00:29:06,606 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for ba4f5827573463818d730cc47ef7a77e: 2024-11-13T00:29:06,606 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-13T00:29:06,606 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-13T00:29:06,639 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42745/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897/wal.1731457745970, size=0 (0bytes) 2024-11-13T00:29:06,639 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:42745/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897/wal.1731457745970 might be still open, length is 0 2024-11-13T00:29:06,639 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42745/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897/wal.1731457745970 2024-11-13T00:29:06,640 WARN [IPC Server handler 0 on default port 42745 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897/wal.1731457745970 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-13T00:29:06,640 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42745/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897/wal.1731457745970 after 1ms 2024-11-13T00:29:07,601 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-13T00:29:07,602 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-13T00:29:07,603 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-13T00:29:07,603 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-13T00:29:09,117 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:55826 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:40949:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55826 dst: /127.0.0.1:40949 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40949 remote=/127.0.0.1:55826]. Total timeout mills is 60000, 57489 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:29:09,117 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:43354 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:41563:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43354 dst: /127.0.0.1:41563 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:29:09,117 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:44260 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:36087:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44260 dst: /127.0.0.1:36087 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:29:09,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741917_1101 (size=1659) 2024-11-13T00:29:09,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741917_1101 (size=1659) 2024-11-13T00:29:09,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741917_1101 (size=1659) 2024-11-13T00:29:10,440 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T00:29:10,641 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42745/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897/wal.1731457745970 after 4002ms 2024-11-13T00:29:10,644 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42745/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897/wal.1731457745970 took 4005ms 2024-11-13T00:29:10,646 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:42745/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897/wal.1731457745970; continuing. 2024-11-13T00:29:10,646 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42745/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897/wal.1731457745970 so closing down 2024-11-13T00:29:10,646 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T00:29:10,648 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731457745970.temp 2024-11-13T00:29:10,649 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/recovered.edits/0000000000000000003-wal.1731457745970.temp 2024-11-13T00:29:10,649 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T00:29:10,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741923_1102 (size=698) 2024-11-13T00:29:10,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741923_1102 (size=698) 2024-11-13T00:29:10,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741923_1102 (size=698) 2024-11-13T00:29:10,663 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/recovered.edits/0000000000000000003-wal.1731457745970.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T00:29:10,665 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/recovered.edits/0000000000000000003-wal.1731457745970.temp to hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/recovered.edits/0000000000000000008 2024-11-13T00:29:10,665 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 20 ms; skipped=3; WAL=hdfs://localhost:42745/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897/wal.1731457745970, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T00:29:10,665 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42745/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897/wal.1731457745970, journal: Splitting hdfs://localhost:42745/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897/wal.1731457745970, size=0 (0bytes) at 1731457746639Finishing writing output for hdfs://localhost:42745/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897/wal.1731457745970 so closing down at 1731457750646 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/recovered.edits/0000000000000000003-wal.1731457745970.temp at 1731457750649 (+3 ms)3 split writer threads finished at 1731457750649Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/recovered.edits/0000000000000000003-wal.1731457745970.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731457750663 (+14 ms)Rename recovered edits hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/recovered.edits/0000000000000000003-wal.1731457745970.temp to hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/recovered.edits/0000000000000000008 at 1731457750665 (+2 ms)Processed 5 edits across 1 Regions in 20 ms; skipped=3; WAL=hdfs://localhost:42745/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897/wal.1731457745970, size=0, length=0, corrupted=false, cancelled=false at 1731457750665 2024-11-13T00:29:10,666 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42745/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897/wal.1731457745970 to hdfs://localhost:42745/hbase/oldWALs/wal.1731457745970 2024-11-13T00:29:10,667 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/recovered.edits/0000000000000000008 2024-11-13T00:29:10,667 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T00:29:10,669 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:29:10,682 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897/wal.1731457750669, exclude list is [], retry=0 2024-11-13T00:29:10,684 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:29:10,685 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:29:10,685 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:29:10,687 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897/wal.1731457750669 2024-11-13T00:29:10,687 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:45019:45019)] 2024-11-13T00:29:10,687 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ba4f5827573463818d730cc47ef7a77e, NAME => 'testCompactedBulkLoadedFiles,,1731457745899.ba4f5827573463818d730cc47ef7a77e.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:29:10,687 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731457745899.ba4f5827573463818d730cc47ef7a77e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:29:10,687 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:10,687 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:10,689 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:10,690 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba4f5827573463818d730cc47ef7a77e columnFamilyName a 2024-11-13T00:29:10,690 DEBUG [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:10,697 DEBUG [StoreFileOpener-ba4f5827573463818d730cc47ef7a77e-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 859be860796a44098f51cdc0d8d25b1a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T00:29:10,697 DEBUG [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/859be860796a44098f51cdc0d8d25b1a_SeqId_4_ 2024-11-13T00:29:10,700 DEBUG [StoreFileOpener-ba4f5827573463818d730cc47ef7a77e-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for be6ebe73891e4f24acf7249beabea75a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T00:29:10,700 DEBUG [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/be6ebe73891e4f24acf7249beabea75a_SeqId_4_ 2024-11-13T00:29:10,704 DEBUG [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/cd541bdc67fe47428ca1bcf3290b0ec2 2024-11-13T00:29:10,707 DEBUG [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/de153366d6fe450eb7f24f087110f936 2024-11-13T00:29:10,710 DEBUG [StoreFileOpener-ba4f5827573463818d730cc47ef7a77e-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for efed154da6d94c2589e2b6e4d45dbfc1_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T00:29:10,710 DEBUG [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/efed154da6d94c2589e2b6e4d45dbfc1_SeqId_4_ 2024-11-13T00:29:10,710 WARN [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/859be860796a44098f51cdc0d8d25b1a_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2948446 2024-11-13T00:29:10,710 WARN [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/be6ebe73891e4f24acf7249beabea75a_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2948446 2024-11-13T00:29:10,710 WARN [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/cd541bdc67fe47428ca1bcf3290b0ec2 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2948446 2024-11-13T00:29:10,710 WARN [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/efed154da6d94c2589e2b6e4d45dbfc1_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2948446 2024-11-13T00:29:10,710 DEBUG [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/859be860796a44098f51cdc0d8d25b1a_SeqId_4_, hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/be6ebe73891e4f24acf7249beabea75a_SeqId_4_, hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/cd541bdc67fe47428ca1bcf3290b0ec2, hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/efed154da6d94c2589e2b6e4d45dbfc1_SeqId_4_] to archive 2024-11-13T00:29:10,711 DEBUG [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T00:29:10,713 DEBUG [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/859be860796a44098f51cdc0d8d25b1a_SeqId_4_ to hdfs://localhost:42745/hbase/archive/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/859be860796a44098f51cdc0d8d25b1a_SeqId_4_ 2024-11-13T00:29:10,714 DEBUG [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/be6ebe73891e4f24acf7249beabea75a_SeqId_4_ to hdfs://localhost:42745/hbase/archive/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/be6ebe73891e4f24acf7249beabea75a_SeqId_4_ 2024-11-13T00:29:10,715 DEBUG [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/cd541bdc67fe47428ca1bcf3290b0ec2 to hdfs://localhost:42745/hbase/archive/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/cd541bdc67fe47428ca1bcf3290b0ec2 2024-11-13T00:29:10,716 DEBUG [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/efed154da6d94c2589e2b6e4d45dbfc1_SeqId_4_ to hdfs://localhost:42745/hbase/archive/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/a/efed154da6d94c2589e2b6e4d45dbfc1_SeqId_4_ 2024-11-13T00:29:10,716 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.HStore(327): Store=ba4f5827573463818d730cc47ef7a77e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:10,717 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:10,717 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba4f5827573463818d730cc47ef7a77e columnFamilyName b 2024-11-13T00:29:10,717 DEBUG [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:10,718 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.HStore(327): Store=ba4f5827573463818d730cc47ef7a77e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:10,718 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:10,719 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba4f5827573463818d730cc47ef7a77e columnFamilyName c 2024-11-13T00:29:10,719 DEBUG [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:10,719 INFO [StoreOpener-ba4f5827573463818d730cc47ef7a77e-1 {}] regionserver.HStore(327): Store=ba4f5827573463818d730cc47ef7a77e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:10,719 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:10,720 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:10,721 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:10,722 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/recovered.edits/0000000000000000008 2024-11-13T00:29:10,725 DEBUG [Time-limited test {}] regionserver.HRegion(5836): ba4f5827573463818d730cc47ef7a77e : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "ba4f5827573463818d730cc47ef7a77e" family_name: "a" compaction_input: "cd541bdc67fe47428ca1bcf3290b0ec2" compaction_input: "be6ebe73891e4f24acf7249beabea75a_SeqId_4_" compaction_input: "859be860796a44098f51cdc0d8d25b1a_SeqId_4_" compaction_input: "efed154da6d94c2589e2b6e4d45dbfc1_SeqId_4_" compaction_output: "de153366d6fe450eb7f24f087110f936" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731457745899.ba4f5827573463818d730cc47ef7a77e." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-13T00:29:10,725 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-13T00:29:10,725 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/recovered.edits/0000000000000000008 2024-11-13T00:29:10,726 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/recovered.edits/0000000000000000008 2024-11-13T00:29:10,727 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:10,727 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:10,728 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T00:29:10,729 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ba4f5827573463818d730cc47ef7a77e 2024-11-13T00:29:10,732 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testCompactedBulkLoadedFiles/ba4f5827573463818d730cc47ef7a77e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T00:29:10,733 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ba4f5827573463818d730cc47ef7a77e; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70213736, jitterRate=0.04626619815826416}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T00:29:10,733 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ba4f5827573463818d730cc47ef7a77e: Writing region info on filesystem at 1731457750688Initializing all the Stores at 1731457750688Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457750688Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457750688Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457750689 (+1 ms)Cleaning up temporary data from old regions at 1731457750727 (+38 ms)Region opened successfully at 1731457750733 (+6 ms) 2024-11-13T00:29:10,735 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ba4f5827573463818d730cc47ef7a77e, disabling compactions & flushes 2024-11-13T00:29:10,735 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731457745899.ba4f5827573463818d730cc47ef7a77e. 2024-11-13T00:29:10,735 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731457745899.ba4f5827573463818d730cc47ef7a77e. 2024-11-13T00:29:10,735 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731457745899.ba4f5827573463818d730cc47ef7a77e. after waiting 0 ms 2024-11-13T00:29:10,735 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731457745899.ba4f5827573463818d730cc47ef7a77e. 2024-11-13T00:29:10,736 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731457745899.ba4f5827573463818d730cc47ef7a77e. 2024-11-13T00:29:10,736 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ba4f5827573463818d730cc47ef7a77e: Waiting for close lock at 1731457750735Disabling compacts and flushes for region at 1731457750735Disabling writes for close at 1731457750735Writing region close event to WAL at 1731457750736 (+1 ms)Closed at 1731457750736 2024-11-13T00:29:10,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741924_1103 (size=93) 2024-11-13T00:29:10,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741924_1103 (size=93) 2024-11-13T00:29:10,741 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T00:29:10,741 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731457750669) 2024-11-13T00:29:10,759 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=439 (was 425) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:42745 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1710617205_22 at /127.0.0.1:55432 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:42745 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1710617205_22 at /127.0.0.1:35626 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1710617205_22 at /127.0.0.1:57778 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1263) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=207 (was 208), ProcessCount=11 (was 11), AvailableMemoryMB=1360 (was 1383) 2024-11-13T00:29:10,759 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-13T00:29:10,772 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=439, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=207, ProcessCount=11, AvailableMemoryMB=1359 2024-11-13T00:29:10,772 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-13T00:29:10,787 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:29:10,789 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T00:29:10,789 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T00:29:10,791 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-18846218, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/hregion-18846218, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:29:10,805 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-18846218/hregion-18846218.1731457750792, exclude list is [], retry=0 2024-11-13T00:29:10,807 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:29:10,808 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:29:10,808 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:29:10,810 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-18846218/hregion-18846218.1731457750792 2024-11-13T00:29:10,810 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:41441:41441)] 2024-11-13T00:29:10,810 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 22e36983ec289e353d607a759f623135, NAME => 'testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42745/hbase 2024-11-13T00:29:10,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741926_1105 (size=67) 2024-11-13T00:29:10,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741926_1105 (size=67) 2024-11-13T00:29:10,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741926_1105 (size=67) 2024-11-13T00:29:10,819 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:29:10,821 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 22e36983ec289e353d607a759f623135 2024-11-13T00:29:10,822 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22e36983ec289e353d607a759f623135 columnFamilyName a 2024-11-13T00:29:10,822 DEBUG [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:10,823 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(327): Store=22e36983ec289e353d607a759f623135/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:10,823 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 22e36983ec289e353d607a759f623135 2024-11-13T00:29:10,824 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22e36983ec289e353d607a759f623135 columnFamilyName b 2024-11-13T00:29:10,824 DEBUG [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:10,824 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(327): Store=22e36983ec289e353d607a759f623135/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:10,824 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 22e36983ec289e353d607a759f623135 2024-11-13T00:29:10,825 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22e36983ec289e353d607a759f623135 columnFamilyName c 2024-11-13T00:29:10,825 DEBUG [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:10,826 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(327): Store=22e36983ec289e353d607a759f623135/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:10,826 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 22e36983ec289e353d607a759f623135 2024-11-13T00:29:10,827 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135 2024-11-13T00:29:10,827 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135 2024-11-13T00:29:10,828 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 22e36983ec289e353d607a759f623135 2024-11-13T00:29:10,828 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 22e36983ec289e353d607a759f623135 2024-11-13T00:29:10,829 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T00:29:10,830 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 22e36983ec289e353d607a759f623135 2024-11-13T00:29:10,832 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T00:29:10,833 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 22e36983ec289e353d607a759f623135; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65554702, jitterRate=-0.02315881848335266}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T00:29:10,833 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 22e36983ec289e353d607a759f623135: Writing region info on filesystem at 1731457750819Initializing all the Stores at 1731457750820 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457750820Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457750820Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457750820Cleaning up temporary data from old regions at 1731457750828 (+8 ms)Region opened successfully at 1731457750833 (+5 ms) 2024-11-13T00:29:10,833 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 22e36983ec289e353d607a759f623135, disabling compactions & flushes 2024-11-13T00:29:10,833 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135. 2024-11-13T00:29:10,833 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135. 2024-11-13T00:29:10,833 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135. after waiting 0 ms 2024-11-13T00:29:10,833 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135. 2024-11-13T00:29:10,834 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135. 2024-11-13T00:29:10,834 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 22e36983ec289e353d607a759f623135: Waiting for close lock at 1731457750833Disabling compacts and flushes for region at 1731457750833Disabling writes for close at 1731457750833Writing region close event to WAL at 1731457750834 (+1 ms)Closed at 1731457750834 2024-11-13T00:29:10,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741925_1104 (size=93) 2024-11-13T00:29:10,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741925_1104 (size=93) 2024-11-13T00:29:10,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741925_1104 (size=93) 2024-11-13T00:29:10,839 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T00:29:10,839 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-18846218:(num 1731457750792) 2024-11-13T00:29:10,839 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T00:29:10,841 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:29:10,856 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750841, exclude list is [], retry=0 2024-11-13T00:29:10,859 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:29:10,859 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:29:10,859 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:29:10,861 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750841 2024-11-13T00:29:10,861 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:45019:45019)] 2024-11-13T00:29:10,861 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 22e36983ec289e353d607a759f623135, NAME => 'testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:29:10,862 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:29:10,862 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 22e36983ec289e353d607a759f623135 2024-11-13T00:29:10,862 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 22e36983ec289e353d607a759f623135 2024-11-13T00:29:10,863 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 22e36983ec289e353d607a759f623135 2024-11-13T00:29:10,864 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22e36983ec289e353d607a759f623135 columnFamilyName a 2024-11-13T00:29:10,864 DEBUG [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:10,865 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(327): Store=22e36983ec289e353d607a759f623135/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:10,865 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 22e36983ec289e353d607a759f623135 2024-11-13T00:29:10,865 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22e36983ec289e353d607a759f623135 columnFamilyName b 2024-11-13T00:29:10,865 DEBUG [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:10,866 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(327): Store=22e36983ec289e353d607a759f623135/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:10,866 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 22e36983ec289e353d607a759f623135 2024-11-13T00:29:10,866 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22e36983ec289e353d607a759f623135 columnFamilyName c 2024-11-13T00:29:10,866 DEBUG [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:10,867 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(327): Store=22e36983ec289e353d607a759f623135/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:10,867 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 22e36983ec289e353d607a759f623135 2024-11-13T00:29:10,868 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135 2024-11-13T00:29:10,869 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135 2024-11-13T00:29:10,870 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 22e36983ec289e353d607a759f623135 2024-11-13T00:29:10,870 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 22e36983ec289e353d607a759f623135 2024-11-13T00:29:10,870 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T00:29:10,872 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 22e36983ec289e353d607a759f623135 2024-11-13T00:29:10,873 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 22e36983ec289e353d607a759f623135; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64253442, jitterRate=-0.04254910349845886}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T00:29:10,873 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 22e36983ec289e353d607a759f623135: Writing region info on filesystem at 1731457750862Initializing all the Stores at 1731457750863 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457750863Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457750863Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457750863Cleaning up temporary data from old regions at 1731457750870 (+7 ms)Region opened successfully at 1731457750873 (+3 ms) 2024-11-13T00:29:10,880 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 22e36983ec289e353d607a759f623135 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-13T00:29:10,896 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/.tmp/a/16072755bef24a79b11c6ab9e03f0ec3 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731457750873/Put/seqid=0 2024-11-13T00:29:10,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741928_1107 (size=5958) 2024-11-13T00:29:10,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741928_1107 (size=5958) 2024-11-13T00:29:10,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741928_1107 (size=5958) 2024-11-13T00:29:10,903 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/.tmp/a/16072755bef24a79b11c6ab9e03f0ec3 2024-11-13T00:29:10,908 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/.tmp/a/16072755bef24a79b11c6ab9e03f0ec3 as hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/a/16072755bef24a79b11c6ab9e03f0ec3 2024-11-13T00:29:10,913 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/a/16072755bef24a79b11c6ab9e03f0ec3, entries=10, sequenceid=13, filesize=5.8 K 2024-11-13T00:29:10,914 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 22e36983ec289e353d607a759f623135 in 34ms, sequenceid=13, compaction requested=false 2024-11-13T00:29:10,914 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 22e36983ec289e353d607a759f623135: 2024-11-13T00:29:10,934 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 22e36983ec289e353d607a759f623135, disabling compactions & flushes 2024-11-13T00:29:10,934 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135. 2024-11-13T00:29:10,935 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135. 2024-11-13T00:29:10,935 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135. after waiting 0 ms 2024-11-13T00:29:10,935 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135. 2024-11-13T00:29:10,936 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135. 2024-11-13T00:29:10,936 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135. 2024-11-13T00:29:10,936 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 22e36983ec289e353d607a759f623135: Waiting for close lock at 1731457750934Disabling compacts and flushes for region at 1731457750934Disabling writes for close at 1731457750935 (+1 ms)Writing region close event to WAL at 1731457750936 (+1 ms)Closed at 1731457750936 2024-11-13T00:29:10,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741927_1106 (size=5869) 2024-11-13T00:29:10,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741927_1106 (size=5869) 2024-11-13T00:29:10,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741927_1106 (size=5869) 2024-11-13T00:29:10,960 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750841, size=5.7 K (5869bytes) 2024-11-13T00:29:10,960 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750841 2024-11-13T00:29:10,960 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750841 after 0ms 2024-11-13T00:29:10,963 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750841 took 4ms 2024-11-13T00:29:10,965 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750841 so closing down 2024-11-13T00:29:10,965 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T00:29:10,966 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731457750841.temp 2024-11-13T00:29:10,967 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000003-wal.1731457750841.temp 2024-11-13T00:29:10,967 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T00:29:10,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741929_1108 (size=5223) 2024-11-13T00:29:10,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741929_1108 (size=5223) 2024-11-13T00:29:10,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741929_1108 (size=5223) 2024-11-13T00:29:10,980 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000003-wal.1731457750841.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T00:29:10,982 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000003-wal.1731457750841.temp to hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000035 2024-11-13T00:29:10,982 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 19 ms; skipped=2; WAL=hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750841, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-11-13T00:29:10,982 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750841, journal: Splitting hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750841, size=5.7 K (5869bytes) at 1731457750960Finishing writing output for hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750841 so closing down at 1731457750965 (+5 ms)Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000003-wal.1731457750841.temp at 1731457750967 (+2 ms)3 split writer threads finished at 1731457750968 (+1 ms)Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000003-wal.1731457750841.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731457750981 (+13 ms)Rename recovered edits hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000003-wal.1731457750841.temp to hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000035 at 1731457750982 (+1 ms)Processed 32 edits across 1 Regions in 19 ms; skipped=2; WAL=hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750841, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1731457750982 2024-11-13T00:29:10,984 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750841 to hdfs://localhost:42745/hbase/oldWALs/wal.1731457750841 2024-11-13T00:29:10,985 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000035 2024-11-13T00:29:10,985 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T00:29:10,987 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:29:11,008 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750988, exclude list is [], retry=0 2024-11-13T00:29:11,011 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:29:11,012 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:29:11,012 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:29:11,014 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750988 2024-11-13T00:29:11,014 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45019:45019),(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:41441:41441)] 2024-11-13T00:29:11,015 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 22e36983ec289e353d607a759f623135, NAME => 'testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135.', STARTKEY => '', ENDKEY => ''} 2024-11-13T00:29:11,015 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:29:11,015 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 22e36983ec289e353d607a759f623135 2024-11-13T00:29:11,015 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 22e36983ec289e353d607a759f623135 2024-11-13T00:29:11,018 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 22e36983ec289e353d607a759f623135 2024-11-13T00:29:11,018 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22e36983ec289e353d607a759f623135 columnFamilyName a 2024-11-13T00:29:11,019 DEBUG [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:11,024 DEBUG [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/a/16072755bef24a79b11c6ab9e03f0ec3 2024-11-13T00:29:11,024 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(327): Store=22e36983ec289e353d607a759f623135/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:11,024 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 22e36983ec289e353d607a759f623135 2024-11-13T00:29:11,025 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22e36983ec289e353d607a759f623135 columnFamilyName b 2024-11-13T00:29:11,025 DEBUG [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:11,025 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(327): Store=22e36983ec289e353d607a759f623135/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:11,026 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 22e36983ec289e353d607a759f623135 2024-11-13T00:29:11,026 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22e36983ec289e353d607a759f623135 columnFamilyName c 2024-11-13T00:29:11,026 DEBUG [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:11,027 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(327): Store=22e36983ec289e353d607a759f623135/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:11,027 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 22e36983ec289e353d607a759f623135 2024-11-13T00:29:11,027 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135 2024-11-13T00:29:11,028 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135 2024-11-13T00:29:11,029 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000035 2024-11-13T00:29:11,031 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000035 2024-11-13T00:29:11,031 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 22e36983ec289e353d607a759f623135 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-13T00:29:11,046 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/.tmp/b/95f3bc9a050b4f488b238d14407b54d5 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731457750914/Put/seqid=0 2024-11-13T00:29:11,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741931_1110 (size=5958) 2024-11-13T00:29:11,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741931_1110 (size=5958) 2024-11-13T00:29:11,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741931_1110 (size=5958) 2024-11-13T00:29:11,054 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/.tmp/b/95f3bc9a050b4f488b238d14407b54d5 2024-11-13T00:29:11,074 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/.tmp/c/3dfd87faa5e245d98116b3a5a18f9f38 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731457750922/Put/seqid=0 2024-11-13T00:29:11,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741932_1111 (size=5958) 2024-11-13T00:29:11,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741932_1111 (size=5958) 2024-11-13T00:29:11,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741932_1111 (size=5958) 2024-11-13T00:29:11,083 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/.tmp/c/3dfd87faa5e245d98116b3a5a18f9f38 2024-11-13T00:29:11,088 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/.tmp/b/95f3bc9a050b4f488b238d14407b54d5 as hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/b/95f3bc9a050b4f488b238d14407b54d5 2024-11-13T00:29:11,093 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/b/95f3bc9a050b4f488b238d14407b54d5, entries=10, sequenceid=35, filesize=5.8 K 2024-11-13T00:29:11,094 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/.tmp/c/3dfd87faa5e245d98116b3a5a18f9f38 as hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/c/3dfd87faa5e245d98116b3a5a18f9f38 2024-11-13T00:29:11,099 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/c/3dfd87faa5e245d98116b3a5a18f9f38, entries=10, sequenceid=35, filesize=5.8 K 2024-11-13T00:29:11,099 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 22e36983ec289e353d607a759f623135 in 68ms, sequenceid=35, compaction requested=false; wal=null 2024-11-13T00:29:11,100 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000035 2024-11-13T00:29:11,100 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 22e36983ec289e353d607a759f623135 2024-11-13T00:29:11,101 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 22e36983ec289e353d607a759f623135 2024-11-13T00:29:11,101 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T00:29:11,102 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 22e36983ec289e353d607a759f623135 2024-11-13T00:29:11,105 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-13T00:29:11,105 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 22e36983ec289e353d607a759f623135; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72109934, jitterRate=0.0745217502117157}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T00:29:11,106 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 22e36983ec289e353d607a759f623135: Writing region info on filesystem at 1731457751015Initializing all the Stores at 1731457751016 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457751016Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457751017 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457751017Obtaining lock to block concurrent updates at 1731457751031 (+14 ms)Preparing flush snapshotting stores in 22e36983ec289e353d607a759f623135 at 1731457751031Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731457751031Flushing stores of testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135. at 1731457751031Flushing 22e36983ec289e353d607a759f623135/b: creating writer at 1731457751032 (+1 ms)Flushing 22e36983ec289e353d607a759f623135/b: appending metadata at 1731457751046 (+14 ms)Flushing 22e36983ec289e353d607a759f623135/b: closing flushed file at 1731457751046Flushing 22e36983ec289e353d607a759f623135/c: creating writer at 1731457751059 (+13 ms)Flushing 22e36983ec289e353d607a759f623135/c: appending metadata at 1731457751073 (+14 ms)Flushing 22e36983ec289e353d607a759f623135/c: closing flushed file at 1731457751073Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47c8b8d5: reopening flushed file at 1731457751088 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34daabea: reopening flushed file at 1731457751093 (+5 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 22e36983ec289e353d607a759f623135 in 68ms, sequenceid=35, compaction requested=false; wal=null at 1731457751099 (+6 ms)Cleaning up temporary data from old regions at 1731457751101 (+2 ms)Region opened successfully at 1731457751106 (+5 ms) 2024-11-13T00:29:11,184 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750988, size=0 (0bytes) 2024-11-13T00:29:11,184 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750988 might be still open, length is 0 2024-11-13T00:29:11,184 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750988 2024-11-13T00:29:11,184 WARN [IPC Server handler 3 on default port 42745 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750988 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-13T00:29:11,185 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750988 after 1ms 2024-11-13T00:29:12,116 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:57868 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:41563:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57868 dst: /127.0.0.1:41563 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41563 remote=/127.0.0.1:57868]. Total timeout mills is 60000, 59020 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:29:12,117 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:35690 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:36087:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35690 dst: /127.0.0.1:36087 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:29:12,117 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-224038803_22 at /127.0.0.1:55482 [Receiving block BP-1811965340-172.17.0.2-1731457714216:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:40949:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55482 dst: /127.0.0.1:40949 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:29:12,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741930_1112 (size=5215) 2024-11-13T00:29:12,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741930_1112 (size=5215) 2024-11-13T00:29:12,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741930_1112 (size=5215) 2024-11-13T00:29:15,185 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750988 after 4001ms 2024-11-13T00:29:15,188 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750988 took 4004ms 2024-11-13T00:29:15,190 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750988; continuing. 2024-11-13T00:29:15,190 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750988 so closing down 2024-11-13T00:29:15,190 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T00:29:15,191 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731457750988.temp 2024-11-13T00:29:15,192 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000037-wal.1731457750988.temp 2024-11-13T00:29:15,193 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T00:29:15,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741933_1113 (size=5223) 2024-11-13T00:29:15,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741933_1113 (size=5223) 2024-11-13T00:29:15,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741933_1113 (size=5223) 2024-11-13T00:29:15,604 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000037-wal.1731457750988.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T00:29:15,606 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000037-wal.1731457750988.temp to hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000066 2024-11-13T00:29:15,606 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 418 ms; skipped=0; WAL=hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750988, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T00:29:15,606 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750988, journal: Splitting hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750988, size=0 (0bytes) at 1731457751184Finishing writing output for hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750988 so closing down at 1731457755190 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000037-wal.1731457750988.temp at 1731457755193 (+3 ms)3 split writer threads finished at 1731457755193Closed recovered edits writer path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000037-wal.1731457750988.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731457755604 (+411 ms)Rename recovered edits hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000037-wal.1731457750988.temp to hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000066 at 1731457755606 (+2 ms)Processed 30 edits across 1 Regions in 418 ms; skipped=0; WAL=hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750988, size=0, length=0, corrupted=false, cancelled=false at 1731457755606 2024-11-13T00:29:15,608 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750988 to hdfs://localhost:42745/hbase/oldWALs/wal.1731457750988 2024-11-13T00:29:15,609 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000066 2024-11-13T00:29:15,609 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T00:29:15,611 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:42745/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786, archiveDir=hdfs://localhost:42745/hbase/oldWALs, maxLogs=32 2024-11-13T00:29:15,627 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457755611, exclude list is [], retry=0 2024-11-13T00:29:15,629 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40949,DS-8d5b419b-bdf7-4a4d-8c0b-7e4a155d465d,DISK] 2024-11-13T00:29:15,630 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36087,DS-31f4be6a-b3ae-4f6d-830c-64ced140ce34,DISK] 2024-11-13T00:29:15,630 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41563,DS-934213d6-8a3f-409f-8a9b-49d4679b1d0e,DISK] 2024-11-13T00:29:15,631 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457755611 2024-11-13T00:29:15,632 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36255:36255),(127.0.0.1/127.0.0.1:41441:41441),(127.0.0.1/127.0.0.1:45019:45019)] 2024-11-13T00:29:15,632 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T00:29:15,633 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 22e36983ec289e353d607a759f623135 2024-11-13T00:29:15,634 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22e36983ec289e353d607a759f623135 columnFamilyName a 2024-11-13T00:29:15,634 DEBUG [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:15,639 DEBUG [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/a/16072755bef24a79b11c6ab9e03f0ec3 2024-11-13T00:29:15,639 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(327): Store=22e36983ec289e353d607a759f623135/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:15,639 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 22e36983ec289e353d607a759f623135 2024-11-13T00:29:15,640 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22e36983ec289e353d607a759f623135 columnFamilyName b 2024-11-13T00:29:15,640 DEBUG [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:15,645 DEBUG [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/b/95f3bc9a050b4f488b238d14407b54d5 2024-11-13T00:29:15,645 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(327): Store=22e36983ec289e353d607a759f623135/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:15,645 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 22e36983ec289e353d607a759f623135 2024-11-13T00:29:15,646 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22e36983ec289e353d607a759f623135 columnFamilyName c 2024-11-13T00:29:15,646 DEBUG [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T00:29:15,650 DEBUG [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/c/3dfd87faa5e245d98116b3a5a18f9f38 2024-11-13T00:29:15,650 INFO [StoreOpener-22e36983ec289e353d607a759f623135-1 {}] regionserver.HStore(327): Store=22e36983ec289e353d607a759f623135/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T00:29:15,651 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 22e36983ec289e353d607a759f623135 2024-11-13T00:29:15,651 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135 2024-11-13T00:29:15,652 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135 2024-11-13T00:29:15,653 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000066 2024-11-13T00:29:15,660 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000066 2024-11-13T00:29:15,660 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 22e36983ec289e353d607a759f623135 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-13T00:29:15,675 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/.tmp/a/7f5bb24c6f0d40259afdf99a5e60af1b is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731457751114/Put/seqid=0 2024-11-13T00:29:15,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741935_1115 (size=5958) 2024-11-13T00:29:15,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741935_1115 (size=5958) 2024-11-13T00:29:15,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741935_1115 (size=5958) 2024-11-13T00:29:15,682 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/.tmp/a/7f5bb24c6f0d40259afdf99a5e60af1b 2024-11-13T00:29:15,708 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/.tmp/b/39796fe534d64abcbb24322934eda6b4 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731457751121/Put/seqid=0 2024-11-13T00:29:15,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741936_1116 (size=5958) 2024-11-13T00:29:15,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741936_1116 (size=5958) 2024-11-13T00:29:15,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741936_1116 (size=5958) 2024-11-13T00:29:15,716 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/.tmp/b/39796fe534d64abcbb24322934eda6b4 2024-11-13T00:29:15,743 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/.tmp/c/bd0a890828e148afaadfe0e2c7d52a9a is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731457751127/Put/seqid=0 2024-11-13T00:29:15,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741937_1117 (size=5958) 2024-11-13T00:29:15,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741937_1117 (size=5958) 2024-11-13T00:29:15,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741937_1117 (size=5958) 2024-11-13T00:29:15,752 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/.tmp/c/bd0a890828e148afaadfe0e2c7d52a9a 2024-11-13T00:29:15,758 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/.tmp/a/7f5bb24c6f0d40259afdf99a5e60af1b as hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/a/7f5bb24c6f0d40259afdf99a5e60af1b 2024-11-13T00:29:15,764 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/a/7f5bb24c6f0d40259afdf99a5e60af1b, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T00:29:15,765 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/.tmp/b/39796fe534d64abcbb24322934eda6b4 as hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/b/39796fe534d64abcbb24322934eda6b4 2024-11-13T00:29:15,771 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/b/39796fe534d64abcbb24322934eda6b4, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T00:29:15,772 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/.tmp/c/bd0a890828e148afaadfe0e2c7d52a9a as hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/c/bd0a890828e148afaadfe0e2c7d52a9a 2024-11-13T00:29:15,777 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/c/bd0a890828e148afaadfe0e2c7d52a9a, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T00:29:15,777 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 22e36983ec289e353d607a759f623135 in 117ms, sequenceid=66, compaction requested=false; wal=null 2024-11-13T00:29:15,778 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/0000000000000000066 2024-11-13T00:29:15,779 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 22e36983ec289e353d607a759f623135 2024-11-13T00:29:15,779 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 22e36983ec289e353d607a759f623135 2024-11-13T00:29:15,780 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T00:29:15,782 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 22e36983ec289e353d607a759f623135 2024-11-13T00:29:15,784 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/hbase/data/default/testReplayEditsWrittenViaHRegion/22e36983ec289e353d607a759f623135/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-13T00:29:15,785 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 22e36983ec289e353d607a759f623135; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67777562, jitterRate=0.00996437668800354}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T00:29:15,785 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 22e36983ec289e353d607a759f623135: Writing region info on filesystem at 1731457755632Initializing all the Stores at 1731457755633 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457755633Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457755633Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731457755633Obtaining lock to block concurrent updates at 1731457755660 (+27 ms)Preparing flush snapshotting stores in 22e36983ec289e353d607a759f623135 at 1731457755660Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731457755660Flushing stores of testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135. at 1731457755660Flushing 22e36983ec289e353d607a759f623135/a: creating writer at 1731457755660Flushing 22e36983ec289e353d607a759f623135/a: appending metadata at 1731457755674 (+14 ms)Flushing 22e36983ec289e353d607a759f623135/a: closing flushed file at 1731457755674Flushing 22e36983ec289e353d607a759f623135/b: creating writer at 1731457755687 (+13 ms)Flushing 22e36983ec289e353d607a759f623135/b: appending metadata at 1731457755707 (+20 ms)Flushing 22e36983ec289e353d607a759f623135/b: closing flushed file at 1731457755707Flushing 22e36983ec289e353d607a759f623135/c: creating writer at 1731457755721 (+14 ms)Flushing 22e36983ec289e353d607a759f623135/c: appending metadata at 1731457755742 (+21 ms)Flushing 22e36983ec289e353d607a759f623135/c: closing flushed file at 1731457755742Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bd35647: reopening flushed file at 1731457755757 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@106ea0aa: reopening flushed file at 1731457755765 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69636340: reopening flushed file at 1731457755771 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 22e36983ec289e353d607a759f623135 in 117ms, sequenceid=66, compaction requested=false; wal=null at 1731457755777 (+6 ms)Cleaning up temporary data from old regions at 1731457755779 (+2 ms)Region opened successfully at 1731457755785 (+6 ms) 2024-11-13T00:29:15,800 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 22e36983ec289e353d607a759f623135, disabling compactions & flushes 2024-11-13T00:29:15,800 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135. 2024-11-13T00:29:15,800 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135. 2024-11-13T00:29:15,800 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135. after waiting 0 ms 2024-11-13T00:29:15,800 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135. 2024-11-13T00:29:15,805 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731457750787.22e36983ec289e353d607a759f623135. 2024-11-13T00:29:15,805 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 22e36983ec289e353d607a759f623135: Waiting for close lock at 1731457755800Disabling compacts and flushes for region at 1731457755800Disabling writes for close at 1731457755800Writing region close event to WAL at 1731457755805 (+5 ms)Closed at 1731457755805 2024-11-13T00:29:15,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741934_1114 (size=93) 2024-11-13T00:29:15,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741934_1114 (size=93) 2024-11-13T00:29:15,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741934_1114 (size=93) 2024-11-13T00:29:15,811 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T00:29:15,811 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731457755611) 2024-11-13T00:29:15,831 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=444 (was 439) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:42745 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-157836313_22 at /127.0.0.1:57912 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:42745 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-157836313_22 at /127.0.0.1:35740 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-157836313_22 at /127.0.0.1:55528 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1412 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 207), ProcessCount=11 (was 11), AvailableMemoryMB=1283 (was 1359) 2024-11-13T00:29:15,832 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1412 is superior to 1024 2024-11-13T00:29:15,832 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T00:29:15,832 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T00:29:15,832 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T00:29:15,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:29:15,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:29:15,833 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T00:29:15,833 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T00:29:15,833 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1411740622, stopped=false 2024-11-13T00:29:15,834 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=cb28580903fb,44487,1731457717196 2024-11-13T00:29:15,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39009-0x1010eb9fb3c0001, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T00:29:15,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x1010eb9fb3c0002, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T00:29:15,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T00:29:15,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39009-0x1010eb9fb3c0001, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:29:15,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x1010eb9fb3c0002, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:29:15,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:29:15,837 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T00:29:15,837 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38607-0x1010eb9fb3c0002, quorum=127.0.0.1:49723, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T00:29:15,838 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T00:29:15,838 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39009-0x1010eb9fb3c0001, quorum=127.0.0.1:49723, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T00:29:15,838 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T00:29:15,839 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T00:29:15,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:29:15,839 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cb28580903fb,39009,1731457717889' ***** 2024-11-13T00:29:15,839 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T00:29:15,839 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'cb28580903fb,38607,1731457717986' ***** 2024-11-13T00:29:15,839 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T00:29:15,840 INFO [RS:1;cb28580903fb:38607 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T00:29:15,840 INFO [RS:0;cb28580903fb:39009 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T00:29:15,840 INFO [RS:1;cb28580903fb:38607 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T00:29:15,840 INFO [RS:0;cb28580903fb:39009 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T00:29:15,840 INFO [RS:1;cb28580903fb:38607 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T00:29:15,840 INFO [RS:0;cb28580903fb:39009 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T00:29:15,840 INFO [RS:1;cb28580903fb:38607 {}] regionserver.HRegionServer(959): stopping server cb28580903fb,38607,1731457717986 2024-11-13T00:29:15,840 INFO [RS:0;cb28580903fb:39009 {}] regionserver.HRegionServer(3091): Received CLOSE for 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:29:15,840 INFO [RS:1;cb28580903fb:38607 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T00:29:15,840 INFO [RS:0;cb28580903fb:39009 {}] regionserver.HRegionServer(959): stopping server cb28580903fb,39009,1731457717889 2024-11-13T00:29:15,840 INFO [RS:1;cb28580903fb:38607 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;cb28580903fb:38607. 2024-11-13T00:29:15,840 INFO [RS:0;cb28580903fb:39009 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T00:29:15,840 DEBUG [RS:1;cb28580903fb:38607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T00:29:15,840 INFO [RS:0;cb28580903fb:39009 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;cb28580903fb:39009. 2024-11-13T00:29:15,840 DEBUG [RS:1;cb28580903fb:38607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:29:15,840 DEBUG [RS:0;cb28580903fb:39009 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T00:29:15,840 INFO [RS:1;cb28580903fb:38607 {}] regionserver.HRegionServer(976): stopping server cb28580903fb,38607,1731457717986; all regions closed. 2024-11-13T00:29:15,840 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1ae5fa43765ef35f2d27709e3e808ead, disabling compactions & flushes 2024-11-13T00:29:15,840 DEBUG [RS:0;cb28580903fb:39009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:29:15,840 INFO [RS:0;cb28580903fb:39009 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T00:29:15,840 INFO [RS:0;cb28580903fb:39009 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T00:29:15,840 INFO [RS:0;cb28580903fb:39009 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T00:29:15,840 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:29:15,841 INFO [RS:0;cb28580903fb:39009 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T00:29:15,841 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:29:15,841 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. after waiting 0 ms 2024-11-13T00:29:15,841 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:29:15,841 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T00:29:15,844 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T00:29:15,845 INFO [RS:0;cb28580903fb:39009 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-13T00:29:15,845 DEBUG [RS:0;cb28580903fb:39009 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 1ae5fa43765ef35f2d27709e3e808ead=testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead.} 2024-11-13T00:29:15,845 DEBUG [RS:0;cb28580903fb:39009 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1ae5fa43765ef35f2d27709e3e808ead 2024-11-13T00:29:15,845 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T00:29:15,845 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T00:29:15,845 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T00:29:15,846 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T00:29:15,846 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T00:29:15,846 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-13T00:29:15,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741834_1010 (size=93) 2024-11-13T00:29:15,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741834_1010 (size=93) 2024-11-13T00:29:15,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741834_1010 (size=93) 2024-11-13T00:29:15,853 DEBUG [RS:1;cb28580903fb:38607 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/oldWALs 2024-11-13T00:29:15,853 INFO [RS:1;cb28580903fb:38607 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cb28580903fb%2C38607%2C1731457717986:(num 1731457719355) 2024-11-13T00:29:15,853 DEBUG [RS:1;cb28580903fb:38607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:29:15,853 INFO [RS:1;cb28580903fb:38607 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T00:29:15,853 INFO [RS:1;cb28580903fb:38607 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T00:29:15,854 INFO [RS:1;cb28580903fb:38607 {}] hbase.ChoreService(370): Chore service for: regionserver/cb28580903fb:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T00:29:15,854 INFO [RS:1;cb28580903fb:38607 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T00:29:15,854 INFO [RS:1;cb28580903fb:38607 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T00:29:15,854 INFO [RS:1;cb28580903fb:38607 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T00:29:15,854 INFO [RS:1;cb28580903fb:38607 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T00:29:15,854 INFO [RS:1;cb28580903fb:38607 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38607 2024-11-13T00:29:15,854 INFO [regionserver/cb28580903fb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T00:29:15,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x1010eb9fb3c0002, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/cb28580903fb,38607,1731457717986 2024-11-13T00:29:15,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T00:29:15,856 INFO [RS:1;cb28580903fb:38607 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T00:29:15,859 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [cb28580903fb,38607,1731457717986] 2024-11-13T00:29:15,861 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/cb28580903fb,38607,1731457717986 already deleted, retry=false 2024-11-13T00:29:15,861 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; cb28580903fb,38607,1731457717986 expired; onlineServers=1 2024-11-13T00:29:15,870 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1ae5fa43765ef35f2d27709e3e808ead/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-13T00:29:15,871 INFO [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:29:15,871 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1ae5fa43765ef35f2d27709e3e808ead: Waiting for close lock at 1731457755840Running coprocessor pre-close hooks at 1731457755840Disabling compacts and flushes for region at 1731457755840Disabling writes for close at 1731457755841 (+1 ms)Writing region close event to WAL at 1731457755865 (+24 ms)Running coprocessor post-close hooks at 1731457755870 (+5 ms)Closed at 1731457755871 (+1 ms) 2024-11-13T00:29:15,871 DEBUG [RS_CLOSE_REGION-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead. 2024-11-13T00:29:15,887 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740/.tmp/info/f219d09c174345d7b154526b3702a8b7 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731457731935.1ae5fa43765ef35f2d27709e3e808ead./info:regioninfo/1731457734892/Put/seqid=0 2024-11-13T00:29:15,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741938_1118 (size=8243) 2024-11-13T00:29:15,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741938_1118 (size=8243) 2024-11-13T00:29:15,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741938_1118 (size=8243) 2024-11-13T00:29:15,899 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740/.tmp/info/f219d09c174345d7b154526b3702a8b7 2024-11-13T00:29:15,920 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740/.tmp/ns/1024e87c550f4f878ac19000a8a0eba0 is 43, key is default/ns:d/1731457719885/Put/seqid=0 2024-11-13T00:29:15,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741939_1119 (size=5153) 2024-11-13T00:29:15,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741939_1119 (size=5153) 2024-11-13T00:29:15,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741939_1119 (size=5153) 2024-11-13T00:29:15,931 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740/.tmp/ns/1024e87c550f4f878ac19000a8a0eba0 2024-11-13T00:29:15,939 INFO [regionserver/cb28580903fb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T00:29:15,939 INFO [regionserver/cb28580903fb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T00:29:15,950 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740/.tmp/table/b058f2ff4f4045058eeb321df81b374e is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731457732349/Put/seqid=0 2024-11-13T00:29:15,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741940_1120 (size=5431) 2024-11-13T00:29:15,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741940_1120 (size=5431) 2024-11-13T00:29:15,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741940_1120 (size=5431) 2024-11-13T00:29:15,957 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740/.tmp/table/b058f2ff4f4045058eeb321df81b374e 2024-11-13T00:29:15,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x1010eb9fb3c0002, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T00:29:15,960 INFO [RS:1;cb28580903fb:38607 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T00:29:15,960 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38607-0x1010eb9fb3c0002, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T00:29:15,960 INFO [RS:1;cb28580903fb:38607 {}] regionserver.HRegionServer(1031): Exiting; stopping=cb28580903fb,38607,1731457717986; zookeeper connection closed. 2024-11-13T00:29:15,960 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@fa73a10 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@fa73a10 2024-11-13T00:29:15,964 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740/.tmp/info/f219d09c174345d7b154526b3702a8b7 as hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740/info/f219d09c174345d7b154526b3702a8b7 2024-11-13T00:29:15,969 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740/info/f219d09c174345d7b154526b3702a8b7, entries=18, sequenceid=21, filesize=8.0 K 2024-11-13T00:29:15,969 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740/.tmp/ns/1024e87c550f4f878ac19000a8a0eba0 as hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740/ns/1024e87c550f4f878ac19000a8a0eba0 2024-11-13T00:29:15,974 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740/ns/1024e87c550f4f878ac19000a8a0eba0, entries=2, sequenceid=21, filesize=5.0 K 2024-11-13T00:29:15,975 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740/.tmp/table/b058f2ff4f4045058eeb321df81b374e as hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740/table/b058f2ff4f4045058eeb321df81b374e 2024-11-13T00:29:15,981 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740/table/b058f2ff4f4045058eeb321df81b374e, entries=2, sequenceid=21, filesize=5.3 K 2024-11-13T00:29:15,982 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 136ms, sequenceid=21, compaction requested=false 2024-11-13T00:29:15,987 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-13T00:29:15,988 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T00:29:15,988 INFO [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T00:29:15,988 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731457755845Running coprocessor pre-close hooks at 1731457755845Disabling compacts and flushes for region at 1731457755845Disabling writes for close at 1731457755846 (+1 ms)Obtaining lock to block concurrent updates at 1731457755846Preparing flush snapshotting stores in 1588230740 at 1731457755846Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1731457755846Flushing stores of hbase:meta,,1.1588230740 at 1731457755847 (+1 ms)Flushing 1588230740/info: creating writer at 1731457755847Flushing 1588230740/info: appending metadata at 1731457755887 (+40 ms)Flushing 1588230740/info: closing flushed file at 1731457755887Flushing 1588230740/ns: creating writer at 1731457755905 (+18 ms)Flushing 1588230740/ns: appending metadata at 1731457755919 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731457755919Flushing 1588230740/table: creating writer at 1731457755936 (+17 ms)Flushing 1588230740/table: appending metadata at 1731457755950 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731457755950Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17092dbf: reopening flushed file at 1731457755963 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@257cbd9f: reopening flushed file at 1731457755969 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a7fa614: reopening flushed file at 1731457755975 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 136ms, sequenceid=21, compaction requested=false at 1731457755982 (+7 ms)Writing region close event to WAL at 1731457755984 (+2 ms)Running coprocessor post-close hooks at 1731457755988 (+4 ms)Closed at 1731457755988 2024-11-13T00:29:15,988 DEBUG [RS_CLOSE_META-regionserver/cb28580903fb:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T00:29:16,045 INFO [RS:0;cb28580903fb:39009 {}] regionserver.HRegionServer(976): stopping server cb28580903fb,39009,1731457717889; all regions closed. 2024-11-13T00:29:16,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741836_1012 (size=8899) 2024-11-13T00:29:16,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741836_1012 (size=8899) 2024-11-13T00:29:16,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741836_1012 (size=8899) 2024-11-13T00:29:16,051 DEBUG [RS:0;cb28580903fb:39009 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/oldWALs 2024-11-13T00:29:16,051 INFO [RS:0;cb28580903fb:39009 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cb28580903fb%2C39009%2C1731457717889.meta:.meta(num 1731457719714) 2024-11-13T00:29:16,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741835_1011 (size=3433) 2024-11-13T00:29:16,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741835_1011 (size=3433) 2024-11-13T00:29:16,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741835_1011 (size=3433) 2024-11-13T00:29:16,066 DEBUG [RS:0;cb28580903fb:39009 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/oldWALs 2024-11-13T00:29:16,067 INFO [RS:0;cb28580903fb:39009 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL cb28580903fb%2C39009%2C1731457717889:(num 1731457719355) 2024-11-13T00:29:16,067 DEBUG [RS:0;cb28580903fb:39009 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T00:29:16,067 INFO [RS:0;cb28580903fb:39009 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T00:29:16,067 INFO [RS:0;cb28580903fb:39009 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T00:29:16,067 INFO [RS:0;cb28580903fb:39009 {}] hbase.ChoreService(370): Chore service for: regionserver/cb28580903fb:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T00:29:16,067 INFO [RS:0;cb28580903fb:39009 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T00:29:16,067 INFO [regionserver/cb28580903fb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T00:29:16,067 INFO [RS:0;cb28580903fb:39009 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39009 2024-11-13T00:29:16,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39009-0x1010eb9fb3c0001, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/cb28580903fb,39009,1731457717889 2024-11-13T00:29:16,069 INFO [RS:0;cb28580903fb:39009 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T00:29:16,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T00:29:16,073 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [cb28580903fb,39009,1731457717889] 2024-11-13T00:29:16,074 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/cb28580903fb,39009,1731457717889 already deleted, retry=false 2024-11-13T00:29:16,074 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; cb28580903fb,39009,1731457717889 expired; onlineServers=0 2024-11-13T00:29:16,074 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'cb28580903fb,44487,1731457717196' ***** 2024-11-13T00:29:16,074 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T00:29:16,074 INFO [M:0;cb28580903fb:44487 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T00:29:16,075 INFO [M:0;cb28580903fb:44487 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T00:29:16,075 DEBUG [M:0;cb28580903fb:44487 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T00:29:16,075 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T00:29:16,075 DEBUG [M:0;cb28580903fb:44487 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T00:29:16,075 DEBUG [master/cb28580903fb:0:becomeActiveMaster-HFileCleaner.small.0-1731457718960 {}] cleaner.HFileCleaner(306): Exit Thread[master/cb28580903fb:0:becomeActiveMaster-HFileCleaner.small.0-1731457718960,5,FailOnTimeoutGroup] 2024-11-13T00:29:16,075 DEBUG [master/cb28580903fb:0:becomeActiveMaster-HFileCleaner.large.0-1731457718959 {}] cleaner.HFileCleaner(306): Exit Thread[master/cb28580903fb:0:becomeActiveMaster-HFileCleaner.large.0-1731457718959,5,FailOnTimeoutGroup] 2024-11-13T00:29:16,075 INFO [M:0;cb28580903fb:44487 {}] hbase.ChoreService(370): Chore service for: master/cb28580903fb:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T00:29:16,075 INFO [M:0;cb28580903fb:44487 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T00:29:16,075 DEBUG [M:0;cb28580903fb:44487 {}] master.HMaster(1795): Stopping service threads 2024-11-13T00:29:16,075 INFO [M:0;cb28580903fb:44487 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T00:29:16,075 INFO [M:0;cb28580903fb:44487 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T00:29:16,076 INFO [M:0;cb28580903fb:44487 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T00:29:16,076 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T00:29:16,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T00:29:16,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T00:29:16,077 DEBUG [M:0;cb28580903fb:44487 {}] zookeeper.ZKUtil(347): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T00:29:16,077 WARN [M:0;cb28580903fb:44487 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T00:29:16,078 INFO [M:0;cb28580903fb:44487 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/.lastflushedseqids 2024-11-13T00:29:16,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741941_1121 (size=138) 2024-11-13T00:29:16,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741941_1121 (size=138) 2024-11-13T00:29:16,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741941_1121 (size=138) 2024-11-13T00:29:16,093 INFO [M:0;cb28580903fb:44487 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T00:29:16,094 INFO [M:0;cb28580903fb:44487 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T00:29:16,094 DEBUG [M:0;cb28580903fb:44487 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T00:29:16,094 INFO [M:0;cb28580903fb:44487 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T00:29:16,094 DEBUG [M:0;cb28580903fb:44487 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T00:29:16,094 DEBUG [M:0;cb28580903fb:44487 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T00:29:16,094 DEBUG [M:0;cb28580903fb:44487 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T00:29:16,094 INFO [M:0;cb28580903fb:44487 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-11-13T00:29:16,151 DEBUG [M:0;cb28580903fb:44487 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/30bf08f7144945208a5a37cc9cffb08b is 82, key is hbase:meta,,1/info:regioninfo/1731457719802/Put/seqid=0 2024-11-13T00:29:16,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741942_1122 (size=5672) 2024-11-13T00:29:16,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741942_1122 (size=5672) 2024-11-13T00:29:16,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741942_1122 (size=5672) 2024-11-13T00:29:16,159 INFO [M:0;cb28580903fb:44487 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/30bf08f7144945208a5a37cc9cffb08b 2024-11-13T00:29:16,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39009-0x1010eb9fb3c0001, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T00:29:16,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39009-0x1010eb9fb3c0001, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T00:29:16,173 INFO [RS:0;cb28580903fb:39009 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T00:29:16,173 INFO [RS:0;cb28580903fb:39009 {}] regionserver.HRegionServer(1031): Exiting; stopping=cb28580903fb,39009,1731457717889; zookeeper connection closed. 2024-11-13T00:29:16,173 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7e992d10 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7e992d10 2024-11-13T00:29:16,174 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-13T00:29:16,180 DEBUG [M:0;cb28580903fb:44487 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e6c1d7e10fd2445fa3b927c9f48f4096 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731457732356/Put/seqid=0 2024-11-13T00:29:16,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741943_1123 (size=7755) 2024-11-13T00:29:16,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741943_1123 (size=7755) 2024-11-13T00:29:16,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741943_1123 (size=7755) 2024-11-13T00:29:16,212 INFO [M:0;cb28580903fb:44487 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e6c1d7e10fd2445fa3b927c9f48f4096 2024-11-13T00:29:16,218 INFO [M:0;cb28580903fb:44487 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e6c1d7e10fd2445fa3b927c9f48f4096 2024-11-13T00:29:16,243 DEBUG [M:0;cb28580903fb:44487 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bf2293443ea146558ae1d1be63f221e2 is 69, key is cb28580903fb,38607,1731457717986/rs:state/1731457719116/Put/seqid=0 2024-11-13T00:29:16,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741944_1124 (size=5445) 2024-11-13T00:29:16,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741944_1124 (size=5445) 2024-11-13T00:29:16,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741944_1124 (size=5445) 2024-11-13T00:29:16,255 INFO [M:0;cb28580903fb:44487 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bf2293443ea146558ae1d1be63f221e2 2024-11-13T00:29:16,260 INFO [M:0;cb28580903fb:44487 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bf2293443ea146558ae1d1be63f221e2 2024-11-13T00:29:16,261 DEBUG [M:0;cb28580903fb:44487 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/30bf08f7144945208a5a37cc9cffb08b as hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/30bf08f7144945208a5a37cc9cffb08b 2024-11-13T00:29:16,266 INFO [M:0;cb28580903fb:44487 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/30bf08f7144945208a5a37cc9cffb08b, entries=8, sequenceid=168, filesize=5.5 K 2024-11-13T00:29:16,267 DEBUG [M:0;cb28580903fb:44487 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e6c1d7e10fd2445fa3b927c9f48f4096 as hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e6c1d7e10fd2445fa3b927c9f48f4096 2024-11-13T00:29:16,273 INFO [M:0;cb28580903fb:44487 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e6c1d7e10fd2445fa3b927c9f48f4096 2024-11-13T00:29:16,273 INFO [M:0;cb28580903fb:44487 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e6c1d7e10fd2445fa3b927c9f48f4096, entries=17, sequenceid=168, filesize=7.6 K 2024-11-13T00:29:16,274 DEBUG [M:0;cb28580903fb:44487 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bf2293443ea146558ae1d1be63f221e2 as hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bf2293443ea146558ae1d1be63f221e2 2024-11-13T00:29:16,280 INFO [M:0;cb28580903fb:44487 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bf2293443ea146558ae1d1be63f221e2 2024-11-13T00:29:16,280 INFO [M:0;cb28580903fb:44487 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42745/user/jenkins/test-data/48f1669f-e288-f2d9-259e-c2a1e92e5e7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bf2293443ea146558ae1d1be63f221e2, entries=3, sequenceid=168, filesize=5.3 K 2024-11-13T00:29:16,281 INFO [M:0;cb28580903fb:44487 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 187ms, sequenceid=168, compaction requested=false 2024-11-13T00:29:16,283 INFO [M:0;cb28580903fb:44487 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T00:29:16,283 DEBUG [M:0;cb28580903fb:44487 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731457756094Disabling compacts and flushes for region at 1731457756094Disabling writes for close at 1731457756094Obtaining lock to block concurrent updates at 1731457756094Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731457756094Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1731457756094Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731457756095 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731457756095Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731457756150 (+55 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731457756150Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731457756164 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731457756179 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731457756179Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731457756218 (+39 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731457756242 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731457756242Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dac0ad3: reopening flushed file at 1731457756260 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21c77b6f: reopening flushed file at 1731457756266 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4627a1ce: reopening flushed file at 1731457756273 (+7 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 187ms, sequenceid=168, compaction requested=false at 1731457756281 (+8 ms)Writing region close event to WAL at 1731457756283 (+2 ms)Closed at 1731457756283 2024-11-13T00:29:16,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41563 is added to blk_1073741830_1006 (size=81431) 2024-11-13T00:29:16,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40949 is added to blk_1073741830_1006 (size=81431) 2024-11-13T00:29:16,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36087 is added to blk_1073741830_1006 (size=81431) 2024-11-13T00:29:16,291 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T00:29:16,291 INFO [M:0;cb28580903fb:44487 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T00:29:16,291 INFO [M:0;cb28580903fb:44487 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44487 2024-11-13T00:29:16,292 INFO [M:0;cb28580903fb:44487 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T00:29:16,394 INFO [M:0;cb28580903fb:44487 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T00:29:16,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T00:29:16,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44487-0x1010eb9fb3c0000, quorum=127.0.0.1:49723, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T00:29:16,400 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731457735281/wal.1731457735573 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731457735281/wal.1731457735573 (inode 16655) Holder DFSClient_NONMAPREDUCE_-224038803_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731457735281/wal.1731457735573 (inode 16655) Holder DFSClient_NONMAPREDUCE_-224038803_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T00:29:16,401 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731457727528/wal.1731457727633 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:29:16,401 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731457750786/wal.1731457750988 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:29:16,404 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731457735710/wal.1731457744952 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731457735710/wal.1731457744952 (inode 16678) Holder DFSClient_NONMAPREDUCE_-224038803_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731457735710/wal.1731457744952 (inode 16678) Holder DFSClient_NONMAPREDUCE_-224038803_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T00:29:16,405 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731457720968/wal.1731457721040 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:29:16,407 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731457745174/wal.1731457745712 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731457745174/wal.1731457745712 (inode 16704) Holder DFSClient_NONMAPREDUCE_-224038803_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731457745174/wal.1731457745712 (inode 16704) Holder DFSClient_NONMAPREDUCE_-224038803_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T00:29:16,407 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731457745897/wal.1731457745970 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T00:29:16,410 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal.1731457720834 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal.1731457720834 (inode 16485) Holder DFSClient_NONMAPREDUCE_-224038803_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731457720640/wal.1731457720834 (inode 16485) Holder DFSClient_NONMAPREDUCE_-224038803_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T00:29:16,412 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal.1731457720396 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal.1731457720396 (inode 16462) Holder DFSClient_NONMAPREDUCE_-224038803_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731457720157/wal.1731457720396 (inode 16462) Holder DFSClient_NONMAPREDUCE_-224038803_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T00:29:16,417 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3217543c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T00:29:16,419 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T00:29:16,419 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T00:29:16,419 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T00:29:16,419 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/hadoop.log.dir/,STOPPED} 2024-11-13T00:29:16,423 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T00:29:16,423 WARN [BP-1811965340-172.17.0.2-1731457714216 heartbeating to localhost/127.0.0.1:42745 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T00:29:16,423 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T00:29:16,423 WARN [BP-1811965340-172.17.0.2-1731457714216 heartbeating to localhost/127.0.0.1:42745 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1811965340-172.17.0.2-1731457714216 (Datanode Uuid e923ddbf-6d36-4714-8dbb-0d1a2e2b0299) service to localhost/127.0.0.1:42745 2024-11-13T00:29:16,425 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/data/data5/current/BP-1811965340-172.17.0.2-1731457714216 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T00:29:16,425 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/data/data6/current/BP-1811965340-172.17.0.2-1731457714216 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T00:29:16,426 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T00:29:16,430 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7e9308b2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T00:29:16,430 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T00:29:16,430 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T00:29:16,430 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T00:29:16,431 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/hadoop.log.dir/,STOPPED} 2024-11-13T00:29:16,432 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T00:29:16,432 WARN [BP-1811965340-172.17.0.2-1731457714216 heartbeating to localhost/127.0.0.1:42745 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T00:29:16,432 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T00:29:16,432 WARN [BP-1811965340-172.17.0.2-1731457714216 heartbeating to localhost/127.0.0.1:42745 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1811965340-172.17.0.2-1731457714216 (Datanode Uuid ac55950c-9dd3-493f-88df-bc11dac90a6f) service to localhost/127.0.0.1:42745 2024-11-13T00:29:16,433 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/data/data3/current/BP-1811965340-172.17.0.2-1731457714216 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T00:29:16,433 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/data/data4/current/BP-1811965340-172.17.0.2-1731457714216 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T00:29:16,434 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T00:29:16,438 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@54387323{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T00:29:16,438 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T00:29:16,438 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T00:29:16,438 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T00:29:16,438 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/hadoop.log.dir/,STOPPED} 2024-11-13T00:29:16,440 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T00:29:16,440 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T00:29:16,440 WARN [BP-1811965340-172.17.0.2-1731457714216 heartbeating to localhost/127.0.0.1:42745 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T00:29:16,440 WARN [BP-1811965340-172.17.0.2-1731457714216 heartbeating to localhost/127.0.0.1:42745 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1811965340-172.17.0.2-1731457714216 (Datanode Uuid d618c93e-4a46-4ed3-8257-ff90f02032ae) service to localhost/127.0.0.1:42745 2024-11-13T00:29:16,441 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/data/data1/current/BP-1811965340-172.17.0.2-1731457714216 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T00:29:16,441 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/cluster_332b13a1-a9e4-7fcd-eba0-b44e5f958c3b/data/data2/current/BP-1811965340-172.17.0.2-1731457714216 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T00:29:16,441 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T00:29:16,448 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@398d2d49{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T00:29:16,449 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T00:29:16,449 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T00:29:16,449 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T00:29:16,449 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4be83919-ebb0-15bd-f318-6f626c4f540d/hadoop.log.dir/,STOPPED} 2024-11-13T00:29:16,460 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T00:29:16,517 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down